Package portage :: Package _emirrordist :: Module FetchIterator
[hide private]

Source Code for Module portage._emirrordist.FetchIterator

  1  # Copyright 2013 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from portage import os 
  5  from portage.checksum import (_apply_hash_filter, 
  6          _filter_unaccelarated_hashes, _hash_filter) 
  7  from portage.dep import use_reduce 
  8  from portage.exception import PortageException 
  9  from .FetchTask import FetchTask 
 10   
11 -class FetchIterator(object):
12
13 - def __init__(self, config):
14 self._config = config 15 self._log_failure = config.log_failure
16
17 - def _iter_every_cp(self):
18 # List categories individually, in order to start yielding quicker, 19 # and in order to reduce latency in case of a signal interrupt. 20 cp_all = self._config.portdb.cp_all 21 for category in sorted(self._config.portdb.categories): 22 for cp in cp_all(categories=(category,)): 23 yield cp
24
25 - def __iter__(self):
26 27 portdb = self._config.portdb 28 get_repo_for_location = portdb.repositories.get_repo_for_location 29 file_owners = self._config.file_owners 30 file_failures = self._config.file_failures 31 restrict_mirror_exemptions = self._config.restrict_mirror_exemptions 32 33 hash_filter = _hash_filter( 34 portdb.settings.get("PORTAGE_CHECKSUM_FILTER", "")) 35 if hash_filter.transparent: 36 hash_filter = None 37 38 for cp in self._iter_every_cp(): 39 40 for tree in portdb.porttrees: 41 42 # Reset state so the Manifest is pulled once 43 # for this cp / tree combination. 44 digests = None 45 repo_config = get_repo_for_location(tree) 46 47 for cpv in portdb.cp_list(cp, mytree=tree): 48 49 try: 50 restrict, = portdb.aux_get(cpv, ("RESTRICT",), 51 mytree=tree) 52 except (KeyError, PortageException) as e: 53 self._log_failure("%s\t\taux_get exception %s" % 54 (cpv, e)) 55 continue 56 57 # Here we use matchnone=True to ignore conditional parts 58 # of RESTRICT since they don't apply unconditionally. 59 # Assume such conditionals only apply on the client side. 60 try: 61 restrict = frozenset(use_reduce(restrict, 62 flat=True, matchnone=True)) 63 except PortageException as e: 64 self._log_failure("%s\t\tuse_reduce exception %s" % 65 (cpv, e)) 66 continue 67 68 if "fetch" in restrict: 69 continue 70 71 try: 72 uri_map = portdb.getFetchMap(cpv) 73 except PortageException as e: 74 self._log_failure("%s\t\tgetFetchMap exception %s" % 75 (cpv, e)) 76 continue 77 78 if not uri_map: 79 continue 80 81 if "mirror" in restrict: 82 skip = False 83 if restrict_mirror_exemptions is not None: 84 new_uri_map = {} 85 for filename, uri_tuple in uri_map.items(): 86 for uri in uri_tuple: 87 if uri[:9] == "mirror://": 88 i = uri.find("/", 9) 89 if i != -1 and uri[9:i].strip("/") in \ 90 restrict_mirror_exemptions: 91 new_uri_map[filename] = uri_tuple 92 break 93 if new_uri_map: 94 uri_map = new_uri_map 95 else: 96 skip = True 97 else: 98 skip = True 99 100 if skip: 101 continue 102 103 # Parse Manifest for this cp if we haven't yet. 104 if digests is None: 105 try: 106 digests = repo_config.load_manifest( 107 os.path.join(repo_config.location, cp) 108 ).getTypeDigests("DIST") 109 except (EnvironmentError, PortageException) as e: 110 for filename in uri_map: 111 self._log_failure( 112 "%s\t%s\tManifest exception %s" % 113 (cpv, filename, e)) 114 file_failures[filename] = cpv 115 continue 116 117 if not digests: 118 for filename in uri_map: 119 self._log_failure("%s\t%s\tdigest entry missing" % 120 (cpv, filename)) 121 file_failures[filename] = cpv 122 continue 123 124 for filename, uri_tuple in uri_map.items(): 125 file_digests = digests.get(filename) 126 if file_digests is None: 127 self._log_failure("%s\t%s\tdigest entry missing" % 128 (cpv, filename)) 129 file_failures[filename] = cpv 130 continue 131 if filename in file_owners: 132 continue 133 file_owners[filename] = cpv 134 135 file_digests = \ 136 _filter_unaccelarated_hashes(file_digests) 137 if hash_filter is not None: 138 file_digests = _apply_hash_filter( 139 file_digests, hash_filter) 140 141 yield FetchTask(cpv=cpv, 142 background=True, 143 digests=file_digests, 144 distfile=filename, 145 restrict=restrict, 146 uri_tuple=uri_tuple, 147 config=self._config)
148