Package portage :: Package cache :: Module sqlite
[hide private]

Source Code for Module portage.cache.sqlite

  1  # Copyright 1999-2014 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from __future__ import division, unicode_literals 
  5   
  6  import re 
  7  import sys 
  8  from portage.cache import fs_template 
  9  from portage.cache import cache_errors 
 10  from portage import os 
 11  from portage import _unicode_decode 
 12  from portage.util import writemsg 
 13  from portage.localization import _ 
 14   
 15  if sys.hexversion >= 0x3000000: 
 16          # pylint: disable=W0622 
 17          basestring = str 
 18   
19 -class database(fs_template.FsBased):
20 21 validation_chf = 'md5' 22 chf_types = ('md5', 'mtime') 23 24 autocommits = False 25 synchronous = False 26 # cache_bytes is used together with page_size (set at sqlite build time) 27 # to calculate the number of pages requested, according to the following 28 # equation: cache_bytes = page_bytes * page_count 29 cache_bytes = 1024 * 1024 * 10 30
31 - def __init__(self, *args, **config):
32 super(database, self).__init__(*args, **config) 33 self._import_sqlite() 34 self._allowed_keys = ["_eclasses_"] 35 self._allowed_keys.extend(self._known_keys) 36 self._allowed_keys.extend('_%s_' % k for k in self.chf_types) 37 self._allowed_keys_set = frozenset(self._allowed_keys) 38 self._allowed_keys = sorted(self._allowed_keys_set) 39 40 self.location = os.path.join(self.location, 41 self.label.lstrip(os.path.sep).rstrip(os.path.sep)) 42 43 if not self.readonly and not os.path.exists(self.location): 44 self._ensure_dirs() 45 46 config.setdefault("autocommit", self.autocommits) 47 config.setdefault("cache_bytes", self.cache_bytes) 48 config.setdefault("synchronous", self.synchronous) 49 # Set longer timeout for throwing a "database is locked" exception. 50 # Default timeout in sqlite3 module is 5.0 seconds. 51 config.setdefault("timeout", 15) 52 self._db_init_connection(config) 53 self._db_init_structures()
54
55 - def _import_sqlite(self):
56 # sqlite3 is optional with >=python-2.5 57 try: 58 import sqlite3 as db_module 59 except ImportError as e: 60 raise cache_errors.InitializationError(self.__class__, e) 61 62 self._db_module = db_module 63 self._db_error = db_module.Error
64
65 - def _db_escape_string(self, s):
66 """meta escaping, returns quoted string for use in sql statements""" 67 if not isinstance(s, basestring): 68 # Avoid potential UnicodeEncodeError in python-2.x by 69 # only calling str() when it's absolutely necessary. 70 s = str(s) 71 return "'%s'" % s.replace("'", "''")
72
73 - def _db_init_connection(self, config):
74 self._dbpath = self.location + ".sqlite" 75 #if os.path.exists(self._dbpath): 76 # os.unlink(self._dbpath) 77 connection_kwargs = {} 78 connection_kwargs["timeout"] = config["timeout"] 79 try: 80 if not self.readonly: 81 self._ensure_dirs() 82 self._db_connection = self._db_module.connect( 83 database=_unicode_decode(self._dbpath), **connection_kwargs) 84 self._db_cursor = self._db_connection.cursor() 85 self._db_cursor.execute("PRAGMA encoding = %s" % self._db_escape_string("UTF-8")) 86 if not self.readonly and not self._ensure_access(self._dbpath): 87 raise cache_errors.InitializationError(self.__class__, "can't ensure perms on %s" % self._dbpath) 88 self._db_init_cache_size(config["cache_bytes"]) 89 self._db_init_synchronous(config["synchronous"]) 90 except self._db_error as e: 91 raise cache_errors.InitializationError(self.__class__, e)
92
93 - def _db_init_structures(self):
94 self._db_table = {} 95 self._db_table["packages"] = {} 96 mytable = "portage_packages" 97 self._db_table["packages"]["table_name"] = mytable 98 self._db_table["packages"]["package_id"] = "internal_db_package_id" 99 self._db_table["packages"]["package_key"] = "portage_package_key" 100 create_statement = [] 101 create_statement.append("CREATE TABLE") 102 create_statement.append(mytable) 103 create_statement.append("(") 104 table_parameters = [] 105 table_parameters.append("%s INTEGER PRIMARY KEY AUTOINCREMENT" % self._db_table["packages"]["package_id"]) 106 table_parameters.append("%s TEXT" % self._db_table["packages"]["package_key"]) 107 for k in self._allowed_keys: 108 table_parameters.append("%s TEXT" % k) 109 table_parameters.append("UNIQUE(%s)" % self._db_table["packages"]["package_key"]) 110 create_statement.append(",".join(table_parameters)) 111 create_statement.append(")") 112 113 self._db_table["packages"]["create"] = " ".join(create_statement) 114 115 cursor = self._db_cursor 116 for k, v in self._db_table.items(): 117 if self._db_table_exists(v["table_name"]): 118 create_statement = self._db_table_get_create(v["table_name"]) 119 table_ok, missing_keys = self._db_validate_create_statement(create_statement) 120 if table_ok: 121 if missing_keys: 122 for k in sorted(missing_keys): 123 cursor.execute("ALTER TABLE %s ADD COLUMN %s TEXT" % 124 (self._db_table["packages"]["table_name"], k)) 125 else: 126 writemsg(_("sqlite: dropping old table: %s\n") % v["table_name"]) 127 cursor.execute("DROP TABLE %s" % v["table_name"]) 128 cursor.execute(v["create"]) 129 else: 130 cursor.execute(v["create"])
131
132 - def _db_table_exists(self, table_name):
133 """return true/false dependant on a tbl existing""" 134 cursor = self._db_cursor 135 cursor.execute("SELECT name FROM sqlite_master WHERE type=\"table\" AND name=%s" % \ 136 self._db_escape_string(table_name)) 137 return len(cursor.fetchall()) == 1
138
139 - def _db_table_get_create(self, table_name):
140 """return true/false dependant on a tbl existing""" 141 cursor = self._db_cursor 142 cursor.execute("SELECT sql FROM sqlite_master WHERE name=%s" % \ 143 self._db_escape_string(table_name)) 144 return cursor.fetchall()[0][0]
145
146 - def _db_validate_create_statement(self, statement):
147 missing_keys = None 148 if statement == self._db_table["packages"]["create"]: 149 return True, missing_keys 150 151 m = re.match(r'^\s*CREATE\s*TABLE\s*%s\s*\(\s*%s\s*INTEGER\s*PRIMARY\s*KEY\s*AUTOINCREMENT\s*,(.*)\)\s*$' % 152 (self._db_table["packages"]["table_name"], 153 self._db_table["packages"]["package_id"]), 154 statement) 155 if m is None: 156 return False, missing_keys 157 158 unique_constraints = set([self._db_table["packages"]["package_key"]]) 159 missing_keys = set(self._allowed_keys) 160 unique_re = re.compile(r'^\s*UNIQUE\s*\(\s*(\w*)\s*\)\s*$') 161 column_re = re.compile(r'^\s*(\w*)\s*TEXT\s*$') 162 for x in m.group(1).split(","): 163 m = column_re.match(x) 164 if m is not None: 165 missing_keys.discard(m.group(1)) 166 continue 167 m = unique_re.match(x) 168 if m is not None: 169 unique_constraints.discard(m.group(1)) 170 continue 171 172 if unique_constraints: 173 return False, missing_keys 174 175 return True, missing_keys
176
177 - def _db_init_cache_size(self, cache_bytes):
178 cursor = self._db_cursor 179 cursor.execute("PRAGMA page_size") 180 page_size=int(cursor.fetchone()[0]) 181 # number of pages, sqlite default is 2000 182 cache_size = cache_bytes // page_size 183 cursor.execute("PRAGMA cache_size = %d" % cache_size) 184 cursor.execute("PRAGMA cache_size") 185 actual_cache_size = int(cursor.fetchone()[0]) 186 del cursor 187 if actual_cache_size != cache_size: 188 raise cache_errors.InitializationError(self.__class__,"actual cache_size = "+actual_cache_size+" does does not match requested size of "+cache_size)
189
190 - def _db_init_synchronous(self, synchronous):
191 cursor = self._db_cursor 192 cursor.execute("PRAGMA synchronous = %d" % synchronous) 193 cursor.execute("PRAGMA synchronous") 194 actual_synchronous=int(cursor.fetchone()[0]) 195 del cursor 196 if actual_synchronous!=synchronous: 197 raise cache_errors.InitializationError(self.__class__,"actual synchronous = "+actual_synchronous+" does does not match requested value of "+synchronous)
198
199 - def _getitem(self, cpv):
200 cursor = self._db_cursor 201 cursor.execute("select * from %s where %s=%s" % \ 202 (self._db_table["packages"]["table_name"], 203 self._db_table["packages"]["package_key"], 204 self._db_escape_string(cpv))) 205 result = cursor.fetchall() 206 if len(result) == 1: 207 pass 208 elif len(result) == 0: 209 raise KeyError(cpv) 210 else: 211 raise cache_errors.CacheCorruption(cpv, "key is not unique") 212 result = result[0] 213 d = {} 214 allowed_keys_set = self._allowed_keys_set 215 for column_index, column_info in enumerate(cursor.description): 216 k = column_info[0] 217 if k in allowed_keys_set: 218 v = result[column_index] 219 if v is None: 220 # This happens after a new empty column has been added. 221 v = "" 222 d[k] = v 223 224 return d
225
226 - def _setitem(self, cpv, values):
227 update_statement = [] 228 update_statement.append("REPLACE INTO %s" % self._db_table["packages"]["table_name"]) 229 update_statement.append("(") 230 update_statement.append(','.join([self._db_table["packages"]["package_key"]] + self._allowed_keys)) 231 update_statement.append(")") 232 update_statement.append("VALUES") 233 update_statement.append("(") 234 values_parameters = [] 235 values_parameters.append(self._db_escape_string(cpv)) 236 for k in self._allowed_keys: 237 values_parameters.append(self._db_escape_string(values.get(k, ''))) 238 update_statement.append(",".join(values_parameters)) 239 update_statement.append(")") 240 cursor = self._db_cursor 241 try: 242 s = " ".join(update_statement) 243 cursor.execute(s) 244 except self._db_error as e: 245 writemsg("%s: %s\n" % (cpv, str(e))) 246 raise
247
248 - def commit(self):
249 self._db_connection.commit()
250
251 - def _delitem(self, cpv):
252 cursor = self._db_cursor 253 cursor.execute("DELETE FROM %s WHERE %s=%s" % \ 254 (self._db_table["packages"]["table_name"], 255 self._db_table["packages"]["package_key"], 256 self._db_escape_string(cpv)))
257
258 - def __contains__(self, cpv):
259 cursor = self._db_cursor 260 cursor.execute(" ".join( 261 ["SELECT %s FROM %s" % 262 (self._db_table["packages"]["package_id"], 263 self._db_table["packages"]["table_name"]), 264 "WHERE %s=%s" % ( 265 self._db_table["packages"]["package_key"], 266 self._db_escape_string(cpv))])) 267 result = cursor.fetchall() 268 if len(result) == 0: 269 return False 270 elif len(result) == 1: 271 return True 272 else: 273 raise cache_errors.CacheCorruption(cpv, "key is not unique")
274
275 - def __iter__(self):
276 """generator for walking the dir struct""" 277 cursor = self._db_cursor 278 cursor.execute("SELECT %s FROM %s" % \ 279 (self._db_table["packages"]["package_key"], 280 self._db_table["packages"]["table_name"])) 281 result = cursor.fetchall() 282 key_list = [x[0] for x in result] 283 del result 284 while key_list: 285 yield key_list.pop()
286