Package portage :: Package dbapi :: Module porttree
[hide private]

Source Code for Module portage.dbapi.porttree

   1  # Copyright 1998-2015 Gentoo Foundation 
   2  # Distributed under the terms of the GNU General Public License v2 
   3   
   4  from __future__ import unicode_literals 
   5   
   6  __all__ = [ 
   7          "close_portdbapi_caches", "FetchlistDict", "portagetree", "portdbapi" 
   8  ] 
   9   
  10  import portage 
  11  portage.proxy.lazyimport.lazyimport(globals(), 
  12          'portage.checksum', 
  13          'portage.data:portage_gid,secpass', 
  14          'portage.dbapi.dep_expand:dep_expand', 
  15          'portage.dep:Atom,dep_getkey,match_from_list,use_reduce,_match_slot', 
  16          'portage.package.ebuild.doebuild:doebuild', 
  17          'portage.util:ensure_dirs,shlex_split,writemsg,writemsg_level', 
  18          'portage.util.listdir:listdir', 
  19          'portage.versions:best,catsplit,catpkgsplit,_pkgsplit@pkgsplit,ver_regexp,_pkg_str', 
  20  ) 
  21   
  22  from portage.cache import volatile 
  23  from portage.cache.cache_errors import CacheError 
  24  from portage.cache.mappings import Mapping 
  25  from portage.dbapi import dbapi 
  26  from portage.exception import PortageException, PortageKeyError, \ 
  27          FileNotFound, InvalidAtom, InvalidData, \ 
  28          InvalidDependString, InvalidPackageName 
  29  from portage.localization import _ 
  30   
  31  from portage import eclass_cache, \ 
  32          eapi_is_supported, \ 
  33          _eapi_is_deprecated 
  34  from portage import os 
  35  from portage import _encodings 
  36  from portage import _unicode_encode 
  37  from portage import OrderedDict 
  38  from portage.util._eventloop.EventLoop import EventLoop 
  39  from portage.util._eventloop.global_event_loop import global_event_loop 
  40  from _emerge.EbuildMetadataPhase import EbuildMetadataPhase 
  41   
  42  import os as _os 
  43  import sys 
  44  import traceback 
  45  import warnings 
  46  import errno 
  47  import collections 
  48   
  49  try: 
  50          from urllib.parse import urlparse 
  51  except ImportError: 
  52          from urlparse import urlparse 
  53   
  54  if sys.hexversion >= 0x3000000: 
  55          # pylint: disable=W0622 
  56          basestring = str 
  57          long = int 
58 59 -def close_portdbapi_caches():
60 # The python interpreter does _not_ guarantee that destructors are 61 # called for objects that remain when the interpreter exits, so we 62 # use an atexit hook to call destructors for any global portdbapi 63 # instances that may have been constructed. 64 try: 65 portage._legacy_globals_constructed 66 except AttributeError: 67 pass 68 else: 69 if "db" in portage._legacy_globals_constructed: 70 try: 71 db = portage.db 72 except AttributeError: 73 pass 74 else: 75 if isinstance(db, dict): 76 for x in db.values(): 77 try: 78 if "porttree" in x.lazy_items: 79 continue 80 except (AttributeError, TypeError): 81 continue 82 try: 83 x = x.pop("porttree").dbapi 84 except (AttributeError, KeyError): 85 continue 86 if not isinstance(x, portdbapi): 87 continue 88 x.close_caches()
89 90 portage.process.atexit_register(close_portdbapi_caches)
91 92 # It used to be necessary for API consumers to remove portdbapi instances 93 # from portdbapi_instances, in order to avoid having accumulated instances 94 # consume memory. Now, portdbapi_instances is just an empty dummy list, so 95 # for backward compatibility, ignore ValueError for removal on non-existent 96 # items. 97 -class _dummy_list(list):
98 - def remove(self, item):
99 # TODO: Trigger a DeprecationWarning here, after stable portage 100 # has dummy portdbapi_instances. 101 try: 102 list.remove(self, item) 103 except ValueError: 104 pass
105
106 107 -class _better_cache(object):
108 109 """ 110 The purpose of better_cache is to locate catpkgs in repositories using ``os.listdir()`` as much as possible, which 111 is less expensive IO-wise than exhaustively doing a stat on each repo for a particular catpkg. better_cache stores a 112 list of repos in which particular catpkgs appear. Various dbapi methods use better_cache to locate repositories of 113 interest related to particular catpkg rather than performing an exhaustive scan of all repos/overlays. 114 115 Better_cache.items data may look like this:: 116 117 { "sys-apps/portage" : [ repo1, repo2 ] } 118 119 Without better_cache, Portage will get slower and slower (due to excessive IO) as more overlays are added. 120 121 Also note that it is OK if this cache has some 'false positive' catpkgs in it. We use it to search for specific 122 catpkgs listed in ebuilds. The likelihood of a false positive catpkg in our cache causing a problem is extremely 123 low, because the user of our cache is passing us a catpkg that came from somewhere and has already undergone some 124 validation, and even then will further interrogate the short-list of repos we return to gather more information 125 on the catpkg. 126 127 Thus, the code below is optimized for speed rather than painstaking correctness. I have added a note to 128 ``dbapi.getRepositories()`` to ensure that developers are aware of this just in case. 129 130 The better_cache has been redesigned to perform on-demand scans -- it will only scan a category at a time, as 131 needed. This should further optimize IO performance by not scanning category directories that are not needed by 132 Portage. 133 """ 134
135 - def __init__(self, repositories):
136 self._items = collections.defaultdict(list) 137 self._scanned_cats = set() 138 139 # ordered list of all portree locations we'll scan: 140 self._repo_list = [repo for repo in reversed(list(repositories)) 141 if repo.location is not None]
142
143 - def __getitem__(self, catpkg):
144 result = self._items.get(catpkg) 145 if result is not None: 146 return result 147 148 cat, pkg = catsplit(catpkg) 149 if cat not in self._scanned_cats: 150 self._scan_cat(cat) 151 return self._items[catpkg]
152
153 - def _scan_cat(self, cat):
154 for repo in self._repo_list: 155 cat_dir = repo.location + "/" + cat 156 try: 157 pkg_list = os.listdir(cat_dir) 158 except OSError as e: 159 if e.errno not in (errno.ENOTDIR, errno.ENOENT, errno.ESTALE): 160 raise 161 continue 162 for p in pkg_list: 163 if os.path.isdir(cat_dir + "/" + p): 164 self._items[cat + "/" + p].append(repo) 165 self._scanned_cats.add(cat)
166
167 168 -class portdbapi(dbapi):
169 """this tree will scan a portage directory located at root (passed to init)""" 170 portdbapi_instances = _dummy_list() 171 _use_mutable = True 172 173 @property
174 - def _categories(self):
175 return self.settings.categories
176 177 @property
178 - def porttree_root(self):
179 warnings.warn("portage.dbapi.porttree.portdbapi.porttree_root is deprecated in favor of portage.repository.config.RepoConfig.location " 180 "(available as repositories[repo_name].location attribute of instances of portage.dbapi.porttree.portdbapi class)", 181 DeprecationWarning, stacklevel=2) 182 return self.settings.repositories.mainRepoLocation()
183 184 @property
185 - def eclassdb(self):
186 warnings.warn("portage.dbapi.porttree.portdbapi.eclassdb is deprecated in favor of portage.repository.config.RepoConfig.eclass_db " 187 "(available as repositories[repo_name].eclass_db attribute of instances of portage.dbapi.porttree.portdbapi class)", 188 DeprecationWarning, stacklevel=2) 189 main_repo = self.repositories.mainRepo() 190 if main_repo is None: 191 return None 192 return main_repo.eclass_db
193
194 - def __init__(self, _unused_param=DeprecationWarning, mysettings=None):
195 """ 196 @param _unused_param: deprecated, use mysettings['PORTDIR'] instead 197 @type _unused_param: None 198 @param mysettings: an immutable config instance 199 @type mysettings: portage.config 200 """ 201 202 from portage import config 203 if mysettings: 204 self.settings = mysettings 205 else: 206 from portage import settings 207 self.settings = config(clone=settings) 208 209 if _unused_param is not DeprecationWarning: 210 warnings.warn("The first parameter of the " + \ 211 "portage.dbapi.porttree.portdbapi" + \ 212 " constructor is unused since portage-2.1.8. " + \ 213 "mysettings['PORTDIR'] is used instead.", 214 DeprecationWarning, stacklevel=2) 215 216 self.repositories = self.settings.repositories 217 self.treemap = self.repositories.treemap 218 219 # This is strictly for use in aux_get() doebuild calls when metadata 220 # is generated by the depend phase. It's safest to use a clone for 221 # this purpose because doebuild makes many changes to the config 222 # instance that is passed in. 223 self.doebuild_settings = config(clone=self.settings) 224 self.depcachedir = os.path.realpath(self.settings.depcachedir) 225 226 if os.environ.get("SANDBOX_ON") == "1": 227 # Make api consumers exempt from sandbox violations 228 # when doing metadata cache updates. 229 sandbox_write = os.environ.get("SANDBOX_WRITE", "").split(":") 230 if self.depcachedir not in sandbox_write: 231 sandbox_write.append(self.depcachedir) 232 os.environ["SANDBOX_WRITE"] = \ 233 ":".join(filter(None, sandbox_write)) 234 235 self.porttrees = list(self.settings.repositories.repoLocationList()) 236 237 # This is used as sanity check for aux_get(). If there is no 238 # root eclass dir, we assume that PORTDIR is invalid or 239 # missing. This check allows aux_get() to detect a missing 240 # portage tree and return early by raising a KeyError. 241 self._have_root_eclass_dir = os.path.isdir( 242 os.path.join(self.settings.repositories.mainRepoLocation(), "eclass")) 243 244 #if the portdbapi is "frozen", then we assume that we can cache everything (that no updates to it are happening) 245 self.xcache = {} 246 self.frozen = 0 247 248 #Keep a list of repo names, sorted by priority (highest priority first). 249 self._ordered_repo_name_list = tuple(reversed(self.repositories.prepos_order)) 250 251 self.auxdbmodule = self.settings.load_best_module("portdbapi.auxdbmodule") 252 self.auxdb = {} 253 self._pregen_auxdb = {} 254 # If the current user doesn't have depcachedir write permission, 255 # then the depcachedir cache is kept here read-only access. 256 self._ro_auxdb = {} 257 self._init_cache_dirs() 258 try: 259 depcachedir_st = os.stat(self.depcachedir) 260 depcachedir_w_ok = os.access(self.depcachedir, os.W_OK) 261 except OSError: 262 depcachedir_st = None 263 depcachedir_w_ok = False 264 265 cache_kwargs = {} 266 267 depcachedir_unshared = False 268 if portage.data.secpass < 1 and \ 269 depcachedir_w_ok and \ 270 depcachedir_st is not None and \ 271 os.getuid() == depcachedir_st.st_uid and \ 272 os.getgid() == depcachedir_st.st_gid: 273 # If this user owns depcachedir and is not in the 274 # portage group, then don't bother to set permissions 275 # on cache entries. This makes it possible to run 276 # egencache without any need to be a member of the 277 # portage group. 278 depcachedir_unshared = True 279 else: 280 cache_kwargs.update({ 281 'gid' : portage_gid, 282 'perms' : 0o664 283 }) 284 285 # If secpass < 1, we don't want to write to the cache 286 # since then we won't be able to apply group permissions 287 # to the cache entries/directories. 288 if (secpass < 1 and not depcachedir_unshared) or not depcachedir_w_ok: 289 for x in self.porttrees: 290 self.auxdb[x] = volatile.database( 291 self.depcachedir, x, self._known_keys, 292 **cache_kwargs) 293 try: 294 self._ro_auxdb[x] = self.auxdbmodule(self.depcachedir, x, 295 self._known_keys, readonly=True, **cache_kwargs) 296 except CacheError: 297 pass 298 else: 299 for x in self.porttrees: 300 if x in self.auxdb: 301 continue 302 # location, label, auxdbkeys 303 self.auxdb[x] = self.auxdbmodule( 304 self.depcachedir, x, self._known_keys, **cache_kwargs) 305 if "metadata-transfer" not in self.settings.features: 306 for x in self.porttrees: 307 if x in self._pregen_auxdb: 308 continue 309 cache = self._create_pregen_cache(x) 310 if cache is not None: 311 self._pregen_auxdb[x] = cache 312 # Selectively cache metadata in order to optimize dep matching. 313 self._aux_cache_keys = set( 314 ["DEPEND", "EAPI", "HDEPEND", 315 "INHERITED", "IUSE", "KEYWORDS", "LICENSE", 316 "PDEPEND", "PROPERTIES", "PROVIDE", "RDEPEND", "repository", 317 "RESTRICT", "SLOT", "DEFINED_PHASES", "REQUIRED_USE"]) 318 319 self._aux_cache = {} 320 self._better_cache = None 321 self._broken_ebuilds = set()
322 323 @property
324 - def _event_loop(self):
325 if portage._internal_caller: 326 # For internal portage usage, the global_event_loop is safe. 327 return global_event_loop() 328 else: 329 # For external API consumers, use a local EventLoop, since 330 # we don't want to assume that it's safe to override the 331 # global SIGCHLD handler. 332 return EventLoop(main=False)
333
334 - def _create_pregen_cache(self, tree):
335 conf = self.repositories.get_repo_for_location(tree) 336 cache = conf.get_pregenerated_cache( 337 self._known_keys, readonly=True) 338 if cache is not None: 339 try: 340 cache.ec = self.repositories.get_repo_for_location(tree).eclass_db 341 except AttributeError: 342 pass 343 344 if not cache.complete_eclass_entries: 345 warnings.warn( 346 ("Repository '%s' used deprecated 'pms' cache format. " 347 "Please migrate to 'md5-dict' format.") % (conf.name,), 348 DeprecationWarning) 349 350 return cache
351
352 - def _init_cache_dirs(self):
353 """Create /var/cache/edb/dep and adjust permissions for the portage 354 group.""" 355 356 dirmode = 0o2070 357 modemask = 0o2 358 359 try: 360 ensure_dirs(self.depcachedir, gid=portage_gid, 361 mode=dirmode, mask=modemask) 362 except PortageException: 363 pass
364
365 - def close_caches(self):
366 if not hasattr(self, "auxdb"): 367 # unhandled exception thrown from constructor 368 return 369 for x in self.auxdb: 370 self.auxdb[x].sync() 371 self.auxdb.clear()
372
373 - def flush_cache(self):
374 for x in self.auxdb.values(): 375 x.sync()
376
377 - def findLicensePath(self, license_name):
378 for x in reversed(self.porttrees): 379 license_path = os.path.join(x, "licenses", license_name) 380 if os.access(license_path, os.R_OK): 381 return license_path 382 return None
383
384 - def findname(self,mycpv, mytree = None, myrepo = None):
385 return self.findname2(mycpv, mytree, myrepo)[0]
386
387 - def getRepositoryPath(self, repository_id):
388 """ 389 This function is required for GLEP 42 compliance; given a valid repository ID 390 it must return a path to the repository 391 TreeMap = { id:path } 392 """ 393 return self.treemap.get(repository_id)
394
395 - def getRepositoryName(self, canonical_repo_path):
396 """ 397 This is the inverse of getRepositoryPath(). 398 @param canonical_repo_path: the canonical path of a repository, as 399 resolved by os.path.realpath() 400 @type canonical_repo_path: String 401 @return: The repo_name for the corresponding repository, or None 402 if the path does not correspond a known repository 403 @rtype: String or None 404 """ 405 try: 406 return self.repositories.get_name_for_location(canonical_repo_path) 407 except KeyError: 408 return None
409
410 - def getRepositories(self, catpkg=None):
411 412 """ 413 With catpkg=None, this will return a complete list of repositories in this dbapi. With catpkg set to a value, 414 this method will return a short-list of repositories that contain this catpkg. Use this second approach if 415 possible, to avoid exhaustively searching all repos for a particular catpkg. It's faster for this method to 416 find the catpkg than for you do it yourself. When specifying catpkg, you should have reasonable assurance that 417 the category is valid and PMS-compliant as the caching mechanism we use does not perform validation checks for 418 categories. 419 420 This function is required for GLEP 42 compliance. 421 422 @param catpkg: catpkg for which we want a list of repositories; we'll get a list of all repos containing this 423 catpkg; if None, return a list of all Repositories that contain a particular catpkg. 424 @return: a list of repositories. 425 """ 426 427 if catpkg is not None and self._better_cache is not None: 428 return [repo.name for repo in self._better_cache[catpkg]] 429 return self._ordered_repo_name_list
430
431 - def getMissingRepoNames(self):
432 """ 433 Returns a list of repository paths that lack profiles/repo_name. 434 """ 435 return self.settings.repositories.missing_repo_names
436
437 - def getIgnoredRepos(self):
438 """ 439 Returns a list of repository paths that have been ignored, because 440 another repo with the same name exists. 441 """ 442 return self.settings.repositories.ignored_repos
443
444 - def findname2(self, mycpv, mytree=None, myrepo=None):
445 """ 446 Returns the location of the CPV, and what overlay it was in. 447 Searches overlays first, then PORTDIR; this allows us to return the first 448 matching file. As opposed to starting in portdir and then doing overlays 449 second, we would have to exhaustively search the overlays until we found 450 the file we wanted. 451 If myrepo is not None it will find packages from this repository(overlay) 452 """ 453 if not mycpv: 454 return (None, 0) 455 456 if myrepo is not None: 457 mytree = self.treemap.get(myrepo) 458 if mytree is None: 459 return (None, 0) 460 461 mysplit = mycpv.split("/") 462 psplit = pkgsplit(mysplit[1]) 463 if psplit is None or len(mysplit) != 2: 464 raise InvalidPackageName(mycpv) 465 466 try: 467 cp = mycpv.cp 468 except AttributeError: 469 cp = mysplit[0] + "/" + psplit[0] 470 471 if self._better_cache is None: 472 if mytree: 473 mytrees = [mytree] 474 else: 475 mytrees = reversed(self.porttrees) 476 else: 477 try: 478 repos = self._better_cache[cp] 479 except KeyError: 480 return (None, 0) 481 482 mytrees = [] 483 for repo in repos: 484 if mytree is not None and mytree != repo.location: 485 continue 486 mytrees.append(repo.location) 487 488 # For optimal performace in this hot spot, we do manual unicode 489 # handling here instead of using the wrapped os module. 490 encoding = _encodings['fs'] 491 errors = 'strict' 492 493 relative_path = mysplit[0] + _os.sep + psplit[0] + _os.sep + \ 494 mysplit[1] + ".ebuild" 495 496 for x in mytrees: 497 filename = x + _os.sep + relative_path 498 if _os.access(_unicode_encode(filename, 499 encoding=encoding, errors=errors), _os.R_OK): 500 return (filename, x) 501 return (None, 0)
502
503 - def _write_cache(self, cpv, repo_path, metadata, ebuild_hash):
504 505 try: 506 cache = self.auxdb[repo_path] 507 chf = cache.validation_chf 508 metadata['_%s_' % chf] = getattr(ebuild_hash, chf) 509 except CacheError: 510 # Normally this shouldn't happen, so we'll show 511 # a traceback for debugging purposes. 512 traceback.print_exc() 513 cache = None 514 515 if cache is not None: 516 try: 517 cache[cpv] = metadata 518 except CacheError: 519 # Normally this shouldn't happen, so we'll show 520 # a traceback for debugging purposes. 521 traceback.print_exc()
522
523 - def _pull_valid_cache(self, cpv, ebuild_path, repo_path):
524 try: 525 ebuild_hash = eclass_cache.hashed_path(ebuild_path) 526 # snag mtime since we use it later, and to trigger stat failure 527 # if it doesn't exist 528 ebuild_hash.mtime 529 except FileNotFound: 530 writemsg(_("!!! aux_get(): ebuild for " \ 531 "'%s' does not exist at:\n") % (cpv,), noiselevel=-1) 532 writemsg("!!! %s\n" % ebuild_path, noiselevel=-1) 533 raise PortageKeyError(cpv) 534 535 # Pull pre-generated metadata from the metadata/cache/ 536 # directory if it exists and is valid, otherwise fall 537 # back to the normal writable cache. 538 auxdbs = [] 539 pregen_auxdb = self._pregen_auxdb.get(repo_path) 540 if pregen_auxdb is not None: 541 auxdbs.append(pregen_auxdb) 542 ro_auxdb = self._ro_auxdb.get(repo_path) 543 if ro_auxdb is not None: 544 auxdbs.append(ro_auxdb) 545 auxdbs.append(self.auxdb[repo_path]) 546 eclass_db = self.repositories.get_repo_for_location(repo_path).eclass_db 547 548 for auxdb in auxdbs: 549 try: 550 metadata = auxdb[cpv] 551 except KeyError: 552 continue 553 except CacheError: 554 if not auxdb.readonly: 555 try: 556 del auxdb[cpv] 557 except (KeyError, CacheError): 558 pass 559 continue 560 eapi = metadata.get('EAPI', '').strip() 561 if not eapi: 562 eapi = '0' 563 metadata['EAPI'] = eapi 564 if not eapi_is_supported(eapi): 565 # Since we're supposed to be able to efficiently obtain the 566 # EAPI from _parse_eapi_ebuild_head, we disregard cache entries 567 # for unsupported EAPIs. 568 continue 569 if auxdb.validate_entry(metadata, ebuild_hash, eclass_db): 570 break 571 else: 572 metadata = None 573 574 return (metadata, ebuild_hash)
575
576 - def aux_get(self, mycpv, mylist, mytree=None, myrepo=None):
577 "stub code for returning auxilliary db information, such as SLOT, DEPEND, etc." 578 'input: "sys-apps/foo-1.0",["SLOT","DEPEND","HOMEPAGE"]' 579 'return: ["0",">=sys-libs/bar-1.0","http://www.foo.com"] or raise PortageKeyError if error' 580 cache_me = False 581 if myrepo is not None: 582 mytree = self.treemap.get(myrepo) 583 if mytree is None: 584 raise PortageKeyError(myrepo) 585 586 if mytree is not None and len(self.porttrees) == 1 \ 587 and mytree == self.porttrees[0]: 588 # mytree matches our only tree, so it's safe to 589 # ignore mytree and cache the result 590 mytree = None 591 myrepo = None 592 593 if mytree is None: 594 cache_me = True 595 if mytree is None and not self._known_keys.intersection( 596 mylist).difference(self._aux_cache_keys): 597 aux_cache = self._aux_cache.get(mycpv) 598 if aux_cache is not None: 599 return [aux_cache.get(x, "") for x in mylist] 600 cache_me = True 601 602 try: 603 cat, pkg = mycpv.split("/", 1) 604 except ValueError: 605 # Missing slash. Can't find ebuild so raise PortageKeyError. 606 raise PortageKeyError(mycpv) 607 608 myebuild, mylocation = self.findname2(mycpv, mytree) 609 610 if not myebuild: 611 writemsg("!!! aux_get(): %s\n" % \ 612 _("ebuild not found for '%s'") % mycpv, noiselevel=1) 613 raise PortageKeyError(mycpv) 614 615 mydata, ebuild_hash = self._pull_valid_cache(mycpv, myebuild, mylocation) 616 doregen = mydata is None 617 618 if doregen: 619 if myebuild in self._broken_ebuilds: 620 raise PortageKeyError(mycpv) 621 622 proc = EbuildMetadataPhase(cpv=mycpv, 623 ebuild_hash=ebuild_hash, portdb=self, 624 repo_path=mylocation, scheduler=self._event_loop, 625 settings=self.doebuild_settings) 626 627 proc.start() 628 proc.wait() 629 630 if proc.returncode != os.EX_OK: 631 self._broken_ebuilds.add(myebuild) 632 raise PortageKeyError(mycpv) 633 634 mydata = proc.metadata 635 636 mydata["repository"] = self.repositories.get_name_for_location(mylocation) 637 mydata["_mtime_"] = ebuild_hash.mtime 638 eapi = mydata.get("EAPI") 639 if not eapi: 640 eapi = "0" 641 mydata["EAPI"] = eapi 642 if eapi_is_supported(eapi): 643 mydata["INHERITED"] = " ".join(mydata.get("_eclasses_", [])) 644 645 #finally, we look at our internal cache entry and return the requested data. 646 returnme = [mydata.get(x, "") for x in mylist] 647 648 if cache_me and self.frozen: 649 aux_cache = {} 650 for x in self._aux_cache_keys: 651 aux_cache[x] = mydata.get(x, "") 652 self._aux_cache[mycpv] = aux_cache 653 654 return returnme
655
656 - def getFetchMap(self, mypkg, useflags=None, mytree=None):
657 """ 658 Get the SRC_URI metadata as a dict which maps each file name to a 659 set of alternative URIs. 660 661 @param mypkg: cpv for an ebuild 662 @type mypkg: String 663 @param useflags: a collection of enabled USE flags, for evaluation of 664 conditionals 665 @type useflags: set, or None to enable all conditionals 666 @param mytree: The canonical path of the tree in which the ebuild 667 is located, or None for automatic lookup 668 @type mypkg: String 669 @return: A dict which maps each file name to a set of alternative 670 URIs. 671 @rtype: dict 672 """ 673 674 try: 675 eapi, myuris = self.aux_get(mypkg, 676 ["EAPI", "SRC_URI"], mytree=mytree) 677 except KeyError: 678 # Convert this to an InvalidDependString exception since callers 679 # already handle it. 680 raise portage.exception.InvalidDependString( 681 "getFetchMap(): aux_get() error reading "+mypkg+"; aborting.") 682 683 if not eapi_is_supported(eapi): 684 # Convert this to an InvalidDependString exception 685 # since callers already handle it. 686 raise portage.exception.InvalidDependString( 687 "getFetchMap(): '%s' has unsupported EAPI: '%s'" % \ 688 (mypkg, eapi)) 689 690 return _parse_uri_map(mypkg, {'EAPI':eapi,'SRC_URI':myuris}, 691 use=useflags)
692
693 - def getfetchsizes(self, mypkg, useflags=None, debug=0, myrepo=None):
694 # returns a filename:size dictionnary of remaining downloads 695 myebuild, mytree = self.findname2(mypkg, myrepo=myrepo) 696 if myebuild is None: 697 raise AssertionError(_("ebuild not found for '%s'") % mypkg) 698 pkgdir = os.path.dirname(myebuild) 699 mf = self.repositories.get_repo_for_location( 700 os.path.dirname(os.path.dirname(pkgdir))).load_manifest( 701 pkgdir, self.settings["DISTDIR"]) 702 checksums = mf.getDigests() 703 if not checksums: 704 if debug: 705 writemsg(_("[empty/missing/bad digest]: %s\n") % (mypkg,)) 706 return {} 707 filesdict={} 708 myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree) 709 #XXX: maybe this should be improved: take partial downloads 710 # into account? check checksums? 711 for myfile in myfiles: 712 try: 713 fetch_size = int(checksums[myfile]["size"]) 714 except (KeyError, ValueError): 715 if debug: 716 writemsg(_("[bad digest]: missing %(file)s for %(pkg)s\n") % {"file":myfile, "pkg":mypkg}) 717 continue 718 file_path = os.path.join(self.settings["DISTDIR"], myfile) 719 mystat = None 720 try: 721 mystat = os.stat(file_path) 722 except OSError: 723 pass 724 if mystat is None: 725 existing_size = 0 726 ro_distdirs = self.settings.get("PORTAGE_RO_DISTDIRS") 727 if ro_distdirs is not None: 728 for x in shlex_split(ro_distdirs): 729 try: 730 mystat = os.stat(os.path.join(x, myfile)) 731 except OSError: 732 pass 733 else: 734 if mystat.st_size == fetch_size: 735 existing_size = fetch_size 736 break 737 else: 738 existing_size = mystat.st_size 739 remaining_size = fetch_size - existing_size 740 if remaining_size > 0: 741 # Assume the download is resumable. 742 filesdict[myfile] = remaining_size 743 elif remaining_size < 0: 744 # The existing file is too large and therefore corrupt. 745 filesdict[myfile] = int(checksums[myfile]["size"]) 746 return filesdict
747
748 - def fetch_check(self, mypkg, useflags=None, mysettings=None, all=False, myrepo=None):
749 """ 750 TODO: account for PORTAGE_RO_DISTDIRS 751 """ 752 if all: 753 useflags = None 754 elif useflags is None: 755 if mysettings: 756 useflags = mysettings["USE"].split() 757 if myrepo is not None: 758 mytree = self.treemap.get(myrepo) 759 if mytree is None: 760 return False 761 else: 762 mytree = None 763 764 myfiles = self.getFetchMap(mypkg, useflags=useflags, mytree=mytree) 765 myebuild = self.findname(mypkg, myrepo=myrepo) 766 if myebuild is None: 767 raise AssertionError(_("ebuild not found for '%s'") % mypkg) 768 pkgdir = os.path.dirname(myebuild) 769 mf = self.repositories.get_repo_for_location( 770 os.path.dirname(os.path.dirname(pkgdir))) 771 mf = mf.load_manifest(pkgdir, self.settings["DISTDIR"]) 772 mysums = mf.getDigests() 773 774 failures = {} 775 for x in myfiles: 776 if not mysums or x not in mysums: 777 ok = False 778 reason = _("digest missing") 779 else: 780 try: 781 ok, reason = portage.checksum.verify_all( 782 os.path.join(self.settings["DISTDIR"], x), mysums[x]) 783 except FileNotFound as e: 784 ok = False 785 reason = _("File Not Found: '%s'") % (e,) 786 if not ok: 787 failures[x] = reason 788 if failures: 789 return False 790 return True
791
792 - def cpv_exists(self, mykey, myrepo=None):
793 "Tells us whether an actual ebuild exists on disk (no masking)" 794 cps2 = mykey.split("/") 795 cps = catpkgsplit(mykey, silent=0) 796 if not cps: 797 #invalid cat/pkg-v 798 return 0 799 if self.findname(cps[0] + "/" + cps2[1], myrepo=myrepo): 800 return 1 801 else: 802 return 0
803
804 - def cp_all(self, categories=None, trees=None, reverse=False, sort=True):
805 """ 806 This returns a list of all keys in our tree or trees 807 @param categories: optional list of categories to search or 808 defaults to self.settings.categories 809 @param trees: optional list of trees to search the categories in or 810 defaults to self.porttrees 811 @param reverse: reverse sort order (default is False) 812 @param sort: return sorted results (default is True) 813 @rtype list of [cat/pkg,...] 814 """ 815 d = {} 816 if categories is None: 817 categories = self.settings.categories 818 if trees is None: 819 trees = self.porttrees 820 for x in categories: 821 for oroot in trees: 822 for y in listdir(oroot+"/"+x, EmptyOnError=1, ignorecvs=1, dirsonly=1): 823 try: 824 atom = Atom("%s/%s" % (x, y)) 825 except InvalidAtom: 826 continue 827 if atom != atom.cp: 828 continue 829 d[atom.cp] = None 830 l = list(d) 831 if sort: 832 l.sort(reverse=reverse) 833 return l
834
835 - def cp_list(self, mycp, use_cache=1, mytree=None):
836 # NOTE: Cache can be safely shared with the match cache, since the 837 # match cache uses the result from dep_expand for the cache_key. 838 if self.frozen and mytree is not None \ 839 and len(self.porttrees) == 1 \ 840 and mytree == self.porttrees[0]: 841 # mytree matches our only tree, so it's safe to 842 # ignore mytree and cache the result 843 mytree = None 844 845 if self.frozen and mytree is None: 846 cachelist = self.xcache["cp-list"].get(mycp) 847 if cachelist is not None: 848 # Try to propagate this to the match-all cache here for 849 # repoman since he uses separate match-all caches for each 850 # profile (due to differences in _get_implicit_iuse). 851 self.xcache["match-all"][(mycp, mycp)] = cachelist 852 return cachelist[:] 853 mysplit = mycp.split("/") 854 invalid_category = mysplit[0] not in self._categories 855 d={} 856 if mytree is not None: 857 if isinstance(mytree, basestring): 858 mytrees = [mytree] 859 else: 860 # assume it's iterable 861 mytrees = mytree 862 elif self._better_cache is None: 863 mytrees = self.porttrees 864 else: 865 mytrees = [repo.location for repo in self._better_cache[mycp]] 866 for oroot in mytrees: 867 try: 868 file_list = os.listdir(os.path.join(oroot, mycp)) 869 except OSError: 870 continue 871 for x in file_list: 872 pf = None 873 if x[-7:] == '.ebuild': 874 pf = x[:-7] 875 876 if pf is not None: 877 ps = pkgsplit(pf) 878 if not ps: 879 writemsg(_("\nInvalid ebuild name: %s\n") % \ 880 os.path.join(oroot, mycp, x), noiselevel=-1) 881 continue 882 if ps[0] != mysplit[1]: 883 writemsg(_("\nInvalid ebuild name: %s\n") % \ 884 os.path.join(oroot, mycp, x), noiselevel=-1) 885 continue 886 ver_match = ver_regexp.match("-".join(ps[1:])) 887 if ver_match is None or not ver_match.groups(): 888 writemsg(_("\nInvalid ebuild version: %s\n") % \ 889 os.path.join(oroot, mycp, x), noiselevel=-1) 890 continue 891 d[_pkg_str(mysplit[0]+"/"+pf)] = None 892 if invalid_category and d: 893 writemsg(_("\n!!! '%s' has a category that is not listed in " \ 894 "%setc/portage/categories\n") % \ 895 (mycp, self.settings["PORTAGE_CONFIGROOT"]), noiselevel=-1) 896 mylist = [] 897 else: 898 mylist = list(d) 899 # Always sort in ascending order here since it's handy 900 # and the result can be easily cached and reused. 901 self._cpv_sort_ascending(mylist) 902 if self.frozen and mytree is None: 903 cachelist = mylist[:] 904 self.xcache["cp-list"][mycp] = cachelist 905 self.xcache["match-all"][(mycp, mycp)] = cachelist 906 return mylist
907
908 - def freeze(self):
909 for x in ("bestmatch-visible", "cp-list", "match-all", 910 "match-all-cpv-only", "match-visible", "minimum-all", 911 "minimum-all-ignore-profile", "minimum-visible"): 912 self.xcache[x]={} 913 self.frozen=1 914 self._better_cache = _better_cache(self.repositories)
915
916 - def melt(self):
917 self.xcache = {} 918 self._aux_cache = {} 919 self._better_cache = None 920 self.frozen = 0
921
922 - def xmatch(self,level,origdep,mydep=None,mykey=None,mylist=None):
923 "caching match function; very trick stuff" 924 if level == "list-visible": 925 level = "match-visible" 926 warnings.warn("The 'list-visible' mode of " 927 "portage.dbapi.porttree.portdbapi.xmatch " 928 "has been renamed to match-visible", 929 DeprecationWarning, stacklevel=2) 930 931 if mydep is None: 932 #this stuff only runs on first call of xmatch() 933 #create mydep, mykey from origdep 934 mydep = dep_expand(origdep, mydb=self, settings=self.settings) 935 mykey = mydep.cp 936 937 #if no updates are being made to the tree, we can consult our xcache... 938 cache_key = None 939 if self.frozen: 940 cache_key = (mydep, mydep.unevaluated_atom) 941 try: 942 return self.xcache[level][cache_key][:] 943 except KeyError: 944 pass 945 946 myval = None 947 mytree = None 948 if mydep.repo is not None: 949 mytree = self.treemap.get(mydep.repo) 950 if mytree is None: 951 if level.startswith("match-"): 952 myval = [] 953 else: 954 myval = "" 955 956 if myval is not None: 957 # Unknown repo, empty result. 958 pass 959 elif level == "match-all-cpv-only": 960 # match *all* packages, only against the cpv, in order 961 # to bypass unnecessary cache access for things like IUSE 962 # and SLOT. 963 if mydep == mykey: 964 # Share cache with match-all/cp_list when the result is the 965 # same. Note that this requires that mydep.repo is None and 966 # thus mytree is also None. 967 level = "match-all" 968 myval = self.cp_list(mykey, mytree=mytree) 969 else: 970 myval = match_from_list(mydep, 971 self.cp_list(mykey, mytree=mytree)) 972 973 elif level in ("bestmatch-visible", "match-all", 974 "match-visible", "minimum-all", "minimum-all-ignore-profile", 975 "minimum-visible"): 976 # Find the minimum matching visible version. This is optimized to 977 # minimize the number of metadata accesses (improves performance 978 # especially in cases where metadata needs to be generated). 979 if mydep == mykey: 980 mylist = self.cp_list(mykey, mytree=mytree) 981 else: 982 mylist = match_from_list(mydep, 983 self.cp_list(mykey, mytree=mytree)) 984 985 ignore_profile = level in ("minimum-all-ignore-profile",) 986 visibility_filter = level not in ("match-all", 987 "minimum-all", "minimum-all-ignore-profile") 988 single_match = level not in ("match-all", "match-visible") 989 myval = [] 990 aux_keys = list(self._aux_cache_keys) 991 if level == "bestmatch-visible": 992 iterfunc = reversed 993 else: 994 iterfunc = iter 995 996 if mydep.repo is not None: 997 repos = [mydep.repo] 998 else: 999 # We iterate over self.porttrees, since it's common to 1000 # tweak this attribute in order to adjust match behavior. 1001 repos = [] 1002 for tree in reversed(self.porttrees): 1003 repos.append(self.repositories.get_name_for_location(tree)) 1004 1005 for cpv in iterfunc(mylist): 1006 for repo in repos: 1007 try: 1008 metadata = dict(zip(aux_keys, 1009 self.aux_get(cpv, aux_keys, myrepo=repo))) 1010 except KeyError: 1011 # ebuild not in this repo, or masked by corruption 1012 continue 1013 1014 try: 1015 pkg_str = _pkg_str(cpv, metadata=metadata, 1016 settings=self.settings) 1017 except InvalidData: 1018 continue 1019 1020 if visibility_filter and not self._visible(pkg_str, metadata): 1021 continue 1022 1023 if mydep.slot is not None and \ 1024 not _match_slot(mydep, pkg_str): 1025 continue 1026 1027 if mydep.unevaluated_atom.use is not None and \ 1028 not self._match_use(mydep, pkg_str, metadata, 1029 ignore_profile=ignore_profile): 1030 continue 1031 1032 myval.append(pkg_str) 1033 # only yield a given cpv once 1034 break 1035 1036 if myval and single_match: 1037 break 1038 1039 if single_match: 1040 if myval: 1041 myval = myval[0] 1042 else: 1043 myval = "" 1044 1045 elif level == "bestmatch-list": 1046 #dep match -- find best match but restrict search to sublist 1047 warnings.warn("The 'bestmatch-list' mode of " 1048 "portage.dbapi.porttree.portdbapi.xmatch is deprecated", 1049 DeprecationWarning, stacklevel=2) 1050 myval = best(list(self._iter_match(mydep, mylist))) 1051 elif level == "match-list": 1052 #dep match -- find all matches but restrict search to sublist (used in 2nd half of visible()) 1053 warnings.warn("The 'match-list' mode of " 1054 "portage.dbapi.porttree.portdbapi.xmatch is deprecated", 1055 DeprecationWarning, stacklevel=2) 1056 myval = list(self._iter_match(mydep, mylist)) 1057 else: 1058 raise AssertionError( 1059 "Invalid level argument: '%s'" % level) 1060 1061 if self.frozen: 1062 xcache_this_level = self.xcache.get(level) 1063 if xcache_this_level is not None: 1064 xcache_this_level[cache_key] = myval 1065 if not isinstance(myval, _pkg_str): 1066 myval = myval[:] 1067 1068 return myval
1069
1070 - def match(self, mydep, use_cache=1):
1071 return self.xmatch("match-visible", mydep)
1072
1073 - def gvisible(self, mylist):
1074 warnings.warn("The 'gvisible' method of " 1075 "portage.dbapi.porttree.portdbapi " 1076 "is deprecated", 1077 DeprecationWarning, stacklevel=2) 1078 return list(self._iter_visible(iter(mylist)))
1079
1080 - def visible(self, cpv_iter):
1081 warnings.warn("The 'visible' method of " 1082 "portage.dbapi.porttree.portdbapi " 1083 "is deprecated", 1084 DeprecationWarning, stacklevel=2) 1085 if cpv_iter is None: 1086 return [] 1087 return list(self._iter_visible(iter(cpv_iter)))
1088
1089 - def _iter_visible(self, cpv_iter, myrepo=None):
1090 """ 1091 Return a new list containing only visible packages. 1092 """ 1093 aux_keys = list(self._aux_cache_keys) 1094 metadata = {} 1095 1096 if myrepo is not None: 1097 repos = [myrepo] 1098 else: 1099 # We iterate over self.porttrees, since it's common to 1100 # tweak this attribute in order to adjust match behavior. 1101 repos = [] 1102 for tree in reversed(self.porttrees): 1103 repos.append(self.repositories.get_name_for_location(tree)) 1104 1105 for mycpv in cpv_iter: 1106 for repo in repos: 1107 metadata.clear() 1108 try: 1109 metadata.update(zip(aux_keys, 1110 self.aux_get(mycpv, aux_keys, myrepo=repo))) 1111 except KeyError: 1112 continue 1113 except PortageException as e: 1114 writemsg("!!! Error: aux_get('%s', %s)\n" % 1115 (mycpv, aux_keys), noiselevel=-1) 1116 writemsg("!!! %s\n" % (e,), noiselevel=-1) 1117 del e 1118 continue 1119 1120 if not self._visible(mycpv, metadata): 1121 continue 1122 1123 yield mycpv 1124 # only yield a given cpv once 1125 break
1126
1127 - def _visible(self, cpv, metadata):
1128 eapi = metadata["EAPI"] 1129 if not eapi_is_supported(eapi): 1130 return False 1131 if _eapi_is_deprecated(eapi): 1132 return False 1133 if not metadata["SLOT"]: 1134 return False 1135 1136 settings = self.settings 1137 if settings._getMaskAtom(cpv, metadata): 1138 return False 1139 if settings._getMissingKeywords(cpv, metadata): 1140 return False 1141 if settings.local_config: 1142 metadata['CHOST'] = settings.get('CHOST', '') 1143 if not settings._accept_chost(cpv, metadata): 1144 return False 1145 metadata["USE"] = "" 1146 if "?" in metadata["LICENSE"] or \ 1147 "?" in metadata["PROPERTIES"]: 1148 self.doebuild_settings.setcpv(cpv, mydb=metadata) 1149 metadata['USE'] = self.doebuild_settings['PORTAGE_USE'] 1150 try: 1151 if settings._getMissingLicenses(cpv, metadata): 1152 return False 1153 if settings._getMissingProperties(cpv, metadata): 1154 return False 1155 if settings._getMissingRestrict(cpv, metadata): 1156 return False 1157 except InvalidDependString: 1158 return False 1159 1160 return True
1161
1162 -class portagetree(object):
1163 - def __init__(self, root=DeprecationWarning, virtual=DeprecationWarning, 1164 settings=None):
1165 """ 1166 Constructor for a PortageTree 1167 1168 @param root: deprecated, defaults to settings['ROOT'] 1169 @type root: String/Path 1170 @param virtual: UNUSED 1171 @type virtual: No Idea 1172 @param settings: Portage Configuration object (portage.settings) 1173 @type settings: Instance of portage.config 1174 """ 1175 1176 if settings is None: 1177 settings = portage.settings 1178 self.settings = settings 1179 1180 if root is not DeprecationWarning: 1181 warnings.warn("The root parameter of the " + \ 1182 "portage.dbapi.porttree.portagetree" + \ 1183 " constructor is now unused. Use " + \ 1184 "settings['ROOT'] instead.", 1185 DeprecationWarning, stacklevel=2) 1186 1187 if virtual is not DeprecationWarning: 1188 warnings.warn("The 'virtual' parameter of the " 1189 "portage.dbapi.porttree.portagetree" 1190 " constructor is unused", 1191 DeprecationWarning, stacklevel=2) 1192 1193 self.portroot = settings["PORTDIR"] 1194 self.__virtual = virtual 1195 self.dbapi = portdbapi(mysettings=settings)
1196 1197 @property
1198 - def root(self):
1199 warnings.warn("The root attribute of " + \ 1200 "portage.dbapi.porttree.portagetree" + \ 1201 " is deprecated. Use " + \ 1202 "settings['ROOT'] instead.", 1203 DeprecationWarning, stacklevel=3) 1204 return self.settings['ROOT']
1205 1206 @property
1207 - def virtual(self):
1208 warnings.warn("The 'virtual' attribute of " + \ 1209 "portage.dbapi.porttree.portagetree" + \ 1210 " is deprecated.", 1211 DeprecationWarning, stacklevel=3) 1212 return self.__virtual
1213
1214 - def dep_bestmatch(self,mydep):
1215 "compatibility method" 1216 mymatch = self.dbapi.xmatch("bestmatch-visible",mydep) 1217 if mymatch is None: 1218 return "" 1219 return mymatch
1220
1221 - def dep_match(self,mydep):
1222 "compatibility method" 1223 mymatch = self.dbapi.xmatch("match-visible",mydep) 1224 if mymatch is None: 1225 return [] 1226 return mymatch
1227
1228 - def exists_specific(self,cpv):
1229 return self.dbapi.cpv_exists(cpv)
1230
1231 - def getallnodes(self):
1232 """new behavior: these are all *unmasked* nodes. There may or may not be available 1233 masked package for nodes in this nodes list.""" 1234 return self.dbapi.cp_all()
1235
1236 - def getname(self, pkgname):
1237 "returns file location for this particular package (DEPRECATED)" 1238 if not pkgname: 1239 return "" 1240 mysplit = pkgname.split("/") 1241 psplit = pkgsplit(mysplit[1]) 1242 return "/".join([self.portroot, mysplit[0], psplit[0], mysplit[1]])+".ebuild"
1243
1244 - def getslot(self,mycatpkg):
1245 "Get a slot for a catpkg; assume it exists." 1246 myslot = "" 1247 try: 1248 myslot = self.dbapi._pkg_str(mycatpkg, None).slot 1249 except KeyError: 1250 pass 1251 return myslot
1252
1253 -class FetchlistDict(Mapping):
1254 """ 1255 This provide a mapping interface to retrieve fetch lists. It's used 1256 to allow portage.manifest.Manifest to access fetch lists via a standard 1257 mapping interface rather than use the dbapi directly. 1258 """
1259 - def __init__(self, pkgdir, settings, mydbapi):
1260 """pkgdir is a directory containing ebuilds and settings is passed into 1261 portdbapi.getfetchlist for __getitem__ calls.""" 1262 self.pkgdir = pkgdir 1263 self.cp = os.sep.join(pkgdir.split(os.sep)[-2:]) 1264 self.settings = settings 1265 self.mytree = os.path.realpath(os.path.dirname(os.path.dirname(pkgdir))) 1266 self.portdb = mydbapi
1267
1268 - def __getitem__(self, pkg_key):
1269 """Returns the complete fetch list for a given package.""" 1270 return list(self.portdb.getFetchMap(pkg_key, mytree=self.mytree))
1271
1272 - def __contains__(self, cpv):
1273 return cpv in self.__iter__()
1274
1275 - def has_key(self, pkg_key):
1276 """Returns true if the given package exists within pkgdir.""" 1277 warnings.warn("portage.dbapi.porttree.FetchlistDict.has_key() is " 1278 "deprecated, use the 'in' operator instead", 1279 DeprecationWarning, stacklevel=2) 1280 return pkg_key in self
1281
1282 - def __iter__(self):
1283 return iter(self.portdb.cp_list(self.cp, mytree=self.mytree))
1284
1285 - def __len__(self):
1286 """This needs to be implemented in order to avoid 1287 infinite recursion in some cases.""" 1288 return len(self.portdb.cp_list(self.cp, mytree=self.mytree))
1289
1290 - def keys(self):
1291 """Returns keys for all packages within pkgdir""" 1292 return self.portdb.cp_list(self.cp, mytree=self.mytree)
1293 1294 if sys.hexversion >= 0x3000000: 1295 keys = __iter__
1296
1297 -def _parse_uri_map(cpv, metadata, use=None):
1298 1299 myuris = use_reduce(metadata.get('SRC_URI', ''), 1300 uselist=use, matchall=(use is None), 1301 is_src_uri=True, 1302 eapi=metadata['EAPI']) 1303 1304 uri_map = OrderedDict() 1305 1306 myuris.reverse() 1307 while myuris: 1308 uri = myuris.pop() 1309 if myuris and myuris[-1] == "->": 1310 myuris.pop() 1311 distfile = myuris.pop() 1312 else: 1313 distfile = os.path.basename(uri) 1314 if not distfile: 1315 raise portage.exception.InvalidDependString( 1316 ("getFetchMap(): '%s' SRC_URI has no file " + \ 1317 "name: '%s'") % (cpv, uri)) 1318 1319 uri_set = uri_map.get(distfile) 1320 if uri_set is None: 1321 # Use OrderedDict to preserve order from SRC_URI 1322 # while ensuring uniqueness. 1323 uri_set = OrderedDict() 1324 uri_map[distfile] = uri_set 1325 1326 # SRC_URI may contain a file name with no scheme, and in 1327 # this case it does not belong in uri_set. 1328 if urlparse(uri).scheme: 1329 uri_set[uri] = True 1330 1331 # Convert OrderedDicts to tuples. 1332 for k, v in uri_map.items(): 1333 uri_map[k] = tuple(v) 1334 1335 return uri_map
1336