Package portage :: Package cache :: Module sqlite
[hide private]

Source Code for Module portage.cache.sqlite

  1  # Copyright 1999-2014 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from __future__ import division, unicode_literals 
  5   
  6  import re 
  7  import sys 
  8  from portage.cache import fs_template 
  9  from portage.cache import cache_errors 
 10  from portage import os 
 11  from portage import _unicode_decode 
 12  from portage.util import writemsg 
 13  from portage.localization import _ 
 14   
 15  if sys.hexversion >= 0x3000000: 
 16          # pylint: disable=W0622 
 17          basestring = str 
 18   
19 -class database(fs_template.FsBased):
20 21 autocommits = False 22 synchronous = False 23 # cache_bytes is used together with page_size (set at sqlite build time) 24 # to calculate the number of pages requested, according to the following 25 # equation: cache_bytes = page_bytes * page_count 26 cache_bytes = 1024 * 1024 * 10 27
28 - def __init__(self, *args, **config):
29 super(database, self).__init__(*args, **config) 30 self._import_sqlite() 31 self._allowed_keys = ["_mtime_", "_eclasses_"] 32 self._allowed_keys.extend(self._known_keys) 33 self._allowed_keys.sort() 34 self._allowed_keys_set = frozenset(self._allowed_keys) 35 self.location = os.path.join(self.location, 36 self.label.lstrip(os.path.sep).rstrip(os.path.sep)) 37 38 if not self.readonly and not os.path.exists(self.location): 39 self._ensure_dirs() 40 41 config.setdefault("autocommit", self.autocommits) 42 config.setdefault("cache_bytes", self.cache_bytes) 43 config.setdefault("synchronous", self.synchronous) 44 # Set longer timeout for throwing a "database is locked" exception. 45 # Default timeout in sqlite3 module is 5.0 seconds. 46 config.setdefault("timeout", 15) 47 self._db_init_connection(config) 48 self._db_init_structures()
49
50 - def _import_sqlite(self):
51 # sqlite3 is optional with >=python-2.5 52 try: 53 import sqlite3 as db_module 54 except ImportError as e: 55 raise cache_errors.InitializationError(self.__class__, e) 56 57 self._db_module = db_module 58 self._db_error = db_module.Error
59
60 - def _db_escape_string(self, s):
61 """meta escaping, returns quoted string for use in sql statements""" 62 if not isinstance(s, basestring): 63 # Avoid potential UnicodeEncodeError in python-2.x by 64 # only calling str() when it's absolutely necessary. 65 s = str(s) 66 return "'%s'" % s.replace("'", "''")
67
68 - def _db_init_connection(self, config):
69 self._dbpath = self.location + ".sqlite" 70 #if os.path.exists(self._dbpath): 71 # os.unlink(self._dbpath) 72 connection_kwargs = {} 73 connection_kwargs["timeout"] = config["timeout"] 74 try: 75 if not self.readonly: 76 self._ensure_dirs() 77 self._db_connection = self._db_module.connect( 78 database=_unicode_decode(self._dbpath), **connection_kwargs) 79 self._db_cursor = self._db_connection.cursor() 80 self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) 81 if not self.readonly and not self._ensure_access(self._dbpath): 82 raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath) 83 self._db_init_cache_size(config["cache_bytes"]) 84 self._db_init_synchronous(config["synchronous"]) 85 except self._db_error as e: 86 raise cache_errors.InitializationError(self.__class__, e)
87
88 - def _db_init_structures(self):
89 self._db_table = {} 90 self._db_table["packages"] = {} 91 mytable = "portage_packages" 92 self._db_table["packages"]["table_name"] = mytable 93 self._db_table["packages"]["package_id"] = "internal_db_package_id" 94 self._db_table["packages"]["package_key"] = "portage_package_key" 95 create_statement = [] 96 create_statement.append("CREATE TABLE") 97 create_statement.append(mytable) 98 create_statement.append("(") 99 table_parameters = [] 100 table_parameters.append("%s INTEGER PRIMARY KEY AUTOINCREMENT" % self._db_table["packages"]["package_id"]) 101 table_parameters.append("%s TEXT" % self._db_table["packages"]["package_key"]) 102 for k in self._allowed_keys: 103 table_parameters.append("%s TEXT" % k) 104 table_parameters.append("UNIQUE(%s)" % self._db_table["packages"]["package_key"]) 105 create_statement.append(",".join(table_parameters)) 106 create_statement.append(")") 107 108 self._db_table["packages"]["create"] = " ".join(create_statement) 109 110 cursor = self._db_cursor 111 for k, v in self._db_table.items(): 112 if self._db_table_exists(v["table_name"]): 113 create_statement = self._db_table_get_create(v["table_name"]) 114 table_ok, missing_keys = self._db_validate_create_statement(create_statement) 115 if table_ok: 116 if missing_keys: 117 for k in sorted(missing_keys): 118 cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % 119 (self._db_table["packages"]["table_name"], k)) 120 else: 121 writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) 122 cursor.execute("DROP TABLE %s" % v["table_name"]) 123 cursor.execute(v["create"]) 124 else: 125 cursor.execute(v["create"])
126
127 - def _db_table_exists(self, table_name):
128 """return true/false dependant on a tbl existing""" 129 cursor = self._db_cursor 130 cursor.execute("SELECT name FROM sqlite_master WHERE type=\"table\" AND name=%s" % \ 131 self._db_escape_string(table_name)) 132 return len(cursor.fetchall()) == 1
133
134 - def _db_table_get_create(self, table_name):
135 """return true/false dependant on a tbl existing""" 136 cursor = self._db_cursor 137 cursor.execute("SELECT sql FROM sqlite_master WHERE name=%s" % \ 138 self._db_escape_string(table_name)) 139 return cursor.fetchall()[0][0]
140
141 - def _db_validate_create_statement(self, statement):
142 missing_keys = None 143 if statement == self._db_table["packages"]["create"]: 144 return True, missing_keys 145 146 m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % 147 (self._db_table["packages"]["table_name"], 148 self._db_table["packages"]["package_id"]), 149 statement) 150 if m is None: 151 return False, missing_keys 152 153 unique_constraints = set([self._db_table["packages"]["package_key"]]) 154 missing_keys = set(self._allowed_keys) 155 unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') 156 column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') 157 for x in m.group(1).split(","): 158 m = column_re.match(x) 159 if m is not None: 160 missing_keys.discard(m.group(1)) 161 continue 162 m = unique_re.match(x) 163 if m is not None: 164 unique_constraints.discard(m.group(1)) 165 continue 166 167 if unique_constraints: 168 return False, missing_keys 169 170 return True, missing_keys
171
172 - def _db_init_cache_size(self, cache_bytes):
173 cursor = self._db_cursor 174 cursor.execute("PRAGMA page_size") 175 page_size=int(cursor.fetchone()[0]) 176 # number of pages, sqlite default is 2000 177 cache_size = cache_bytes // page_size 178 cursor.execute("PRAGMA cache_size = %d" % cache_size) 179 cursor.execute("PRAGMA cache_size") 180 actual_cache_size = int(cursor.fetchone()[0]) 181 del cursor 182 if actual_cache_size != cache_size: 183 raise cache_errors.InitializationError(self.__class__,"actual cache_size = "+actual_cache_size+" does does not match requested size of "+cache_size)
184
185 - def _db_init_synchronous(self, synchronous):
186 cursor = self._db_cursor 187 cursor.execute("PRAGMA synchronous = %d" % synchronous) 188 cursor.execute("PRAGMA synchronous") 189 actual_synchronous=int(cursor.fetchone()[0]) 190 del cursor 191 if actual_synchronous!=synchronous: 192 raise cache_errors.InitializationError(self.__class__,"actual synchronous = "+actual_synchronous+" does does not match requested value of "+synchronous)
193
194 - def _getitem(self, cpv):
195 cursor = self._db_cursor 196 cursor.execute("select * from %s where %s=%s" % \ 197 (self._db_table["packages"]["table_name"], 198 self._db_table["packages"]["package_key"], 199 self._db_escape_string(cpv))) 200 result = cursor.fetchall() 201 if len(result) == 1: 202 pass 203 elif len(result) == 0: 204 raise KeyError(cpv) 205 else: 206 raise cache_errors.CacheCorruption(cpv, "key is not unique") 207 result = result[0] 208 d = {} 209 allowed_keys_set = self._allowed_keys_set 210 for column_index, column_info in enumerate(cursor.description): 211 k = column_info[0] 212 if k in allowed_keys_set: 213 v = result[column_index] 214 if v is None: 215 # This happens after a new empty column has been added. 216 v = "" 217 d[k] = v 218 219 return d
220
221 - def _setitem(self, cpv, values):
222 update_statement = [] 223 update_statement.append("REPLACE INTO %s" % self._db_table["packages"]["table_name"]) 224 update_statement.append("(") 225 update_statement.append(','.join([self._db_table["packages"]["package_key"]] + self._allowed_keys)) 226 update_statement.append(")") 227 update_statement.append("VALUES") 228 update_statement.append("(") 229 values_parameters = [] 230 values_parameters.append(self._db_escape_string(cpv)) 231 for k in self._allowed_keys: 232 values_parameters.append(self._db_escape_string(values.get(k, ''))) 233 update_statement.append(",".join(values_parameters)) 234 update_statement.append(")") 235 cursor = self._db_cursor 236 try: 237 s = " ".join(update_statement) 238 cursor.execute(s) 239 except self._db_error as e: 240 writemsg("%s: %s\n" % (cpv, str(e))) 241 raise
242
243 - def commit(self):
244 self._db_connection.commit()
245
246 - def _delitem(self, cpv):
247 cursor = self._db_cursor 248 cursor.execute("DELETE FROM %s WHERE %s=%s" % \ 249 (self._db_table["packages"]["table_name"], 250 self._db_table["packages"]["package_key"], 251 self._db_escape_string(cpv)))
252
253 - def __contains__(self, cpv):
254 cursor = self._db_cursor 255 cursor.execute(" ".join( 256 ["SELECT %s FROM %s" % 257 (self._db_table["packages"]["package_id"], 258 self._db_table["packages"]["table_name"]), 259 "WHERE %s=%s" % ( 260 self._db_table["packages"]["package_key"], 261 self._db_escape_string(cpv))])) 262 result = cursor.fetchall() 263 if len(result) == 0: 264 return False 265 elif len(result) == 1: 266 return True 267 else: 268 raise cache_errors.CacheCorruption(cpv, "key is not unique")
269
270 - def __iter__(self):
271 """generator for walking the dir struct""" 272 cursor = self._db_cursor 273 cursor.execute("SELECT %s FROM %s" % \ 274 (self._db_table["packages"]["package_key"], 275 self._db_table["packages"]["table_name"])) 276 result = cursor.fetchall() 277 key_list = [x[0] for x in result] 278 del result 279 while key_list: 280 yield key_list.pop()
281