Package portage :: Package cache :: Module sqlite
[hide private]

Source Code for Module portage.cache.sqlite

  1  # Copyright 1999-2013 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from __future__ import unicode_literals 
  5   
  6  import re 
  7  import sys 
  8  from portage.cache import fs_template 
  9  from portage.cache import cache_errors 
 10  from portage import os 
 11  from portage import _unicode_decode 
 12  from portage.util import writemsg 
 13  from portage.localization import _ 
 14   
 15  if sys.hexversion >= 0x3000000: 
 16          basestring = str 
 17   
18 -class database(fs_template.FsBased):
19 20 autocommits = False 21 synchronous = False 22 # cache_bytes is used together with page_size (set at sqlite build time) 23 # to calculate the number of pages requested, according to the following 24 # equation: cache_bytes = page_bytes * page_count 25 cache_bytes = 1024 * 1024 * 10 26
27 - def __init__(self, *args, **config):
28 super(database, self).__init__(*args, **config) 29 self._import_sqlite() 30 self._allowed_keys = ["_mtime_", "_eclasses_"] 31 self._allowed_keys.extend(self._known_keys) 32 self._allowed_keys.sort() 33 self._allowed_keys_set = frozenset(self._allowed_keys) 34 self.location = os.path.join(self.location, 35 self.label.lstrip(os.path.sep).rstrip(os.path.sep)) 36 37 if not self.readonly and not os.path.exists(self.location): 38 self._ensure_dirs() 39 40 config.setdefault("autocommit", self.autocommits) 41 config.setdefault("cache_bytes", self.cache_bytes) 42 config.setdefault("synchronous", self.synchronous) 43 # Set longer timeout for throwing a "database is locked" exception. 44 # Default timeout in sqlite3 module is 5.0 seconds. 45 config.setdefault("timeout", 15) 46 self._db_init_connection(config) 47 self._db_init_structures()
48
49 - def _import_sqlite(self):
50 # sqlite3 is optional with >=python-2.5 51 try: 52 import sqlite3 as db_module 53 except ImportError as e: 54 raise cache_errors.InitializationError(self.__class__, e) 55 56 self._db_module = db_module 57 self._db_error = db_module.Error
58
59 - def _db_escape_string(self, s):
60 """meta escaping, returns quoted string for use in sql statements""" 61 if not isinstance(s, basestring): 62 # Avoid potential UnicodeEncodeError in python-2.x by 63 # only calling str() when it's absolutely necessary. 64 s = str(s) 65 return "'%s'" % s.replace("'", "''")
66
67 - def _db_init_connection(self, config):
68 self._dbpath = self.location + ".sqlite" 69 #if os.path.exists(self._dbpath): 70 # os.unlink(self._dbpath) 71 connection_kwargs = {} 72 connection_kwargs["timeout"] = config["timeout"] 73 try: 74 if not self.readonly: 75 self._ensure_dirs() 76 self._db_connection = self._db_module.connect( 77 database=_unicode_decode(self._dbpath), **connection_kwargs) 78 self._db_cursor = self._db_connection.cursor() 79 self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) 80 if not self.readonly and not self._ensure_access(self._dbpath): 81 raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath) 82 self._db_init_cache_size(config["cache_bytes"]) 83 self._db_init_synchronous(config["synchronous"]) 84 except self._db_error as e: 85 raise cache_errors.InitializationError(self.__class__, e)
86
87 - def _db_init_structures(self):
88 self._db_table = {} 89 self._db_table["packages"] = {} 90 mytable = "portage_packages" 91 self._db_table["packages"]["table_name"] = mytable 92 self._db_table["packages"]["package_id"] = "internal_db_package_id" 93 self._db_table["packages"]["package_key"] = "portage_package_key" 94 create_statement = [] 95 create_statement.append("CREATE TABLE") 96 create_statement.append(mytable) 97 create_statement.append("(") 98 table_parameters = [] 99 table_parameters.append("%s INTEGER PRIMARY KEY AUTOINCREMENT" % self._db_table["packages"]["package_id"]) 100 table_parameters.append("%s TEXT" % self._db_table["packages"]["package_key"]) 101 for k in self._allowed_keys: 102 table_parameters.append("%s TEXT" % k) 103 table_parameters.append("UNIQUE(%s)" % self._db_table["packages"]["package_key"]) 104 create_statement.append(",".join(table_parameters)) 105 create_statement.append(")") 106 107 self._db_table["packages"]["create"] = " ".join(create_statement) 108 109 cursor = self._db_cursor 110 for k, v in self._db_table.items(): 111 if self._db_table_exists(v["table_name"]): 112 create_statement = self._db_table_get_create(v["table_name"]) 113 table_ok, missing_keys = self._db_validate_create_statement(create_statement) 114 if table_ok: 115 if missing_keys: 116 for k in sorted(missing_keys): 117 cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % 118 (self._db_table["packages"]["table_name"], k)) 119 else: 120 writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) 121 cursor.execute("DROP TABLE %s" % v["table_name"]) 122 cursor.execute(v["create"]) 123 else: 124 cursor.execute(v["create"])
125
126 - def _db_table_exists(self, table_name):
127 """return true/false dependant on a tbl existing""" 128 cursor = self._db_cursor 129 cursor.execute("SELECT name FROM sqlite_master WHERE type=\"table\" AND name=%s" % \ 130 self._db_escape_string(table_name)) 131 return len(cursor.fetchall()) == 1
132
133 - def _db_table_get_create(self, table_name):
134 """return true/false dependant on a tbl existing""" 135 cursor = self._db_cursor 136 cursor.execute("SELECT sql FROM sqlite_master WHERE name=%s" % \ 137 self._db_escape_string(table_name)) 138 return cursor.fetchall()[0][0]
139
140 - def _db_validate_create_statement(self, statement):
141 missing_keys = None 142 if statement == self._db_table["packages"]["create"]: 143 return True, missing_keys 144 145 m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % 146 (self._db_table["packages"]["table_name"], 147 self._db_table["packages"]["package_id"]), 148 statement) 149 if m is None: 150 return False, missing_keys 151 152 unique_constraints = set([self._db_table["packages"]["package_key"]]) 153 missing_keys = set(self._allowed_keys) 154 unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') 155 column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') 156 for x in m.group(1).split(","): 157 m = column_re.match(x) 158 if m is not None: 159 missing_keys.discard(m.group(1)) 160 continue 161 m = unique_re.match(x) 162 if m is not None: 163 unique_constraints.discard(m.group(1)) 164 continue 165 166 if unique_constraints: 167 return False, missing_keys 168 169 return True, missing_keys
170
171 - def _db_init_cache_size(self, cache_bytes):
172 cursor = self._db_cursor 173 cursor.execute("PRAGMA page_size") 174 page_size=int(cursor.fetchone()[0]) 175 # number of pages, sqlite default is 2000 176 cache_size = cache_bytes / page_size 177 cursor.execute("PRAGMA cache_size = %d" % cache_size) 178 cursor.execute("PRAGMA cache_size") 179 actual_cache_size = int(cursor.fetchone()[0]) 180 del cursor 181 if actual_cache_size != cache_size: 182 raise cache_errors.InitializationError(self.__class__,"actual cache_size = "+actual_cache_size+" does does not match requested size of "+cache_size)
183
184 - def _db_init_synchronous(self, synchronous):
185 cursor = self._db_cursor 186 cursor.execute("PRAGMA synchronous = %d" % synchronous) 187 cursor.execute("PRAGMA synchronous") 188 actual_synchronous=int(cursor.fetchone()[0]) 189 del cursor 190 if actual_synchronous!=synchronous: 191 raise cache_errors.InitializationError(self.__class__,"actual synchronous = "+actual_synchronous+" does does not match requested value of "+synchronous)
192
193 - def _getitem(self, cpv):
194 cursor = self._db_cursor 195 cursor.execute("select * from %s where %s=%s" % \ 196 (self._db_table["packages"]["table_name"], 197 self._db_table["packages"]["package_key"], 198 self._db_escape_string(cpv))) 199 result = cursor.fetchall() 200 if len(result) == 1: 201 pass 202 elif len(result) == 0: 203 raise KeyError(cpv) 204 else: 205 raise cache_errors.CacheCorruption(cpv, "key is not unique") 206 result = result[0] 207 d = {} 208 allowed_keys_set = self._allowed_keys_set 209 for column_index, column_info in enumerate(cursor.description): 210 k = column_info[0] 211 if k in allowed_keys_set: 212 v = result[column_index] 213 if v is None: 214 # This happens after a new empty column has been added. 215 v = "" 216 d[k] = v 217 218 return d
219
220 - def _setitem(self, cpv, values):
221 update_statement = [] 222 update_statement.append("REPLACE INTO %s" % self._db_table["packages"]["table_name"]) 223 update_statement.append("(") 224 update_statement.append(','.join([self._db_table["packages"]["package_key"]] + self._allowed_keys)) 225 update_statement.append(")") 226 update_statement.append("VALUES") 227 update_statement.append("(") 228 values_parameters = [] 229 values_parameters.append(self._db_escape_string(cpv)) 230 for k in self._allowed_keys: 231 values_parameters.append(self._db_escape_string(values.get(k, ''))) 232 update_statement.append(",".join(values_parameters)) 233 update_statement.append(")") 234 cursor = self._db_cursor 235 try: 236 s = " ".join(update_statement) 237 cursor.execute(s) 238 except self._db_error as e: 239 writemsg("%s: %s\n" % (cpv, str(e))) 240 raise
241
242 - def commit(self):
243 self._db_connection.commit()
244
245 - def _delitem(self, cpv):
246 cursor = self._db_cursor 247 cursor.execute("DELETE FROM %s WHERE %s=%s" % \ 248 (self._db_table["packages"]["table_name"], 249 self._db_table["packages"]["package_key"], 250 self._db_escape_string(cpv)))
251
252 - def __contains__(self, cpv):
253 cursor = self._db_cursor 254 cursor.execute(" ".join( 255 ["SELECT %s FROM %s" % 256 (self._db_table["packages"]["package_id"], 257 self._db_table["packages"]["table_name"]), 258 "WHERE %s=%s" % ( 259 self._db_table["packages"]["package_key"], 260 self._db_escape_string(cpv))])) 261 result = cursor.fetchall() 262 if len(result) == 0: 263 return False 264 elif len(result) == 1: 265 return True 266 else: 267 raise cache_errors.CacheCorruption(cpv, "key is not unique")
268
269 - def __iter__(self):
270 """generator for walking the dir struct""" 271 cursor = self._db_cursor 272 cursor.execute("SELECT %s FROM %s" % \ 273 (self._db_table["packages"]["package_key"], 274 self._db_table["packages"]["table_name"])) 275 result = cursor.fetchall() 276 key_list = [x[0] for x in result] 277 del result 278 while key_list: 279 yield key_list.pop()
280