Package _emerge :: Module EbuildBuild
[hide private]

Source Code for Module _emerge.EbuildBuild

  1  # Copyright 1999-2014 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from __future__ import unicode_literals 
  5   
  6  import io 
  7   
  8  import _emerge.emergelog 
  9  from _emerge.AsynchronousTask import AsynchronousTask 
 10  from _emerge.EbuildExecuter import EbuildExecuter 
 11  from _emerge.EbuildPhase import EbuildPhase 
 12  from _emerge.EbuildBinpkg import EbuildBinpkg 
 13  from _emerge.EbuildFetcher import EbuildFetcher 
 14  from _emerge.CompositeTask import CompositeTask 
 15  from _emerge.EbuildMerge import EbuildMerge 
 16  from _emerge.EbuildFetchonly import EbuildFetchonly 
 17  from _emerge.EbuildBuildDir import EbuildBuildDir 
 18  from _emerge.MiscFunctionsProcess import MiscFunctionsProcess 
 19  from _emerge.TaskSequence import TaskSequence 
 20   
 21  import portage 
 22  from portage import _encodings, _unicode_decode, _unicode_encode, os 
 23  from portage.package.ebuild.digestcheck import digestcheck 
 24  from portage.package.ebuild.digestgen import digestgen 
 25  from portage.package.ebuild.doebuild import _check_temp_dir 
 26  from portage.package.ebuild._spawn_nofetch import spawn_nofetch 
 27   
28 -class EbuildBuild(CompositeTask):
29 30 __slots__ = ("args_set", "config_pool", "find_blockers", 31 "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", 32 "prefetcher", "settings", "world_atom") + \ 33 ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") 34
35 - def _start(self):
36 37 pkg = self.pkg 38 settings = self.settings 39 40 if not self.opts.fetchonly: 41 rval = _check_temp_dir(settings) 42 if rval != os.EX_OK: 43 self.returncode = rval 44 self._current_task = None 45 self._async_wait() 46 return 47 48 root_config = pkg.root_config 49 tree = "porttree" 50 self._tree = tree 51 portdb = root_config.trees[tree].dbapi 52 settings.setcpv(pkg) 53 settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" 54 if self.opts.buildpkgonly: 55 settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" 56 else: 57 settings.configdict["pkg"]["MERGE_TYPE"] = "source" 58 ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) 59 if ebuild_path is None: 60 raise AssertionError("ebuild not found for '%s'" % pkg.cpv) 61 self._ebuild_path = ebuild_path 62 portage.doebuild_environment(ebuild_path, 'setup', 63 settings=self.settings, db=portdb) 64 65 # Check the manifest here since with --keep-going mode it's 66 # currently possible to get this far with a broken manifest. 67 if not self._check_manifest(): 68 self.returncode = 1 69 self._current_task = None 70 self._async_wait() 71 return 72 73 prefetcher = self.prefetcher 74 if prefetcher is None: 75 pass 76 elif prefetcher.isAlive() and \ 77 prefetcher.poll() is None: 78 79 if not self.background: 80 fetch_log = os.path.join( 81 _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') 82 msg = ( 83 'Fetching files in the background.', 84 'To view fetch progress, run in another terminal:', 85 'tail -f %s' % fetch_log, 86 ) 87 out = portage.output.EOutput() 88 for l in msg: 89 out.einfo(l) 90 91 self._current_task = prefetcher 92 prefetcher.addExitListener(self._prefetch_exit) 93 return 94 95 self._prefetch_exit(prefetcher)
96
97 - def _check_manifest(self):
98 success = True 99 100 settings = self.settings 101 if 'strict' in settings.features and \ 102 'digest' not in settings.features: 103 settings['O'] = os.path.dirname(self._ebuild_path) 104 quiet_setting = settings.get('PORTAGE_QUIET') 105 settings['PORTAGE_QUIET'] = '1' 106 try: 107 success = digestcheck([], settings, strict=True) 108 finally: 109 if quiet_setting: 110 settings['PORTAGE_QUIET'] = quiet_setting 111 else: 112 del settings['PORTAGE_QUIET'] 113 114 return success
115
116 - def _prefetch_exit(self, prefetcher):
117 118 if self._was_cancelled(): 119 self.wait() 120 return 121 122 opts = self.opts 123 pkg = self.pkg 124 settings = self.settings 125 126 if opts.fetchonly: 127 if opts.pretend: 128 fetcher = EbuildFetchonly( 129 fetch_all=opts.fetch_all_uri, 130 pkg=pkg, pretend=opts.pretend, 131 settings=settings) 132 retval = fetcher.execute() 133 self.returncode = retval 134 self.wait() 135 return 136 else: 137 fetcher = EbuildFetcher( 138 config_pool=self.config_pool, 139 ebuild_path=self._ebuild_path, 140 fetchall=self.opts.fetch_all_uri, 141 fetchonly=self.opts.fetchonly, 142 background=False, 143 logfile=None, 144 pkg=self.pkg, 145 scheduler=self.scheduler) 146 self._start_task(fetcher, self._fetchonly_exit) 147 return 148 149 self._build_dir = EbuildBuildDir( 150 scheduler=self.scheduler, settings=settings) 151 self._build_dir.lock() 152 153 # Cleaning needs to happen before fetch, since the build dir 154 # is used for log handling. 155 msg = " === (%s of %s) Cleaning (%s::%s)" % \ 156 (self.pkg_count.curval, self.pkg_count.maxval, 157 self.pkg.cpv, self._ebuild_path) 158 short_msg = "emerge: (%s of %s) %s Clean" % \ 159 (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) 160 self.logger.log(msg, short_msg=short_msg) 161 162 pre_clean_phase = EbuildPhase(background=self.background, 163 phase='clean', scheduler=self.scheduler, settings=self.settings) 164 self._start_task(pre_clean_phase, self._pre_clean_exit)
165
166 - def _fetchonly_exit(self, fetcher):
167 self._final_exit(fetcher) 168 if self.returncode != os.EX_OK: 169 portdb = self.pkg.root_config.trees[self._tree].dbapi 170 spawn_nofetch(portdb, self._ebuild_path, settings=self.settings) 171 elif 'digest' in self.settings.features: 172 if not digestgen(mysettings=self.settings, 173 myportdb=self.pkg.root_config.trees[self._tree].dbapi): 174 self.returncode = 1 175 self.wait()
176
177 - def _pre_clean_exit(self, pre_clean_phase):
178 if self._default_exit(pre_clean_phase) != os.EX_OK: 179 self._unlock_builddir() 180 self.wait() 181 return 182 183 # for log handling 184 portage.prepare_build_dirs(self.pkg.root, self.settings, 1) 185 186 fetcher = EbuildFetcher(config_pool=self.config_pool, 187 ebuild_path=self._ebuild_path, 188 fetchall=self.opts.fetch_all_uri, 189 fetchonly=self.opts.fetchonly, 190 background=self.background, 191 logfile=self.settings.get('PORTAGE_LOG_FILE'), 192 pkg=self.pkg, scheduler=self.scheduler) 193 194 try: 195 already_fetched = fetcher.already_fetched(self.settings) 196 except portage.exception.InvalidDependString as e: 197 msg_lines = [] 198 msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ 199 (self.pkg.cpv, e) 200 msg_lines.append(msg) 201 fetcher._eerror(msg_lines) 202 portage.elog.elog_process(self.pkg.cpv, self.settings) 203 self.returncode = 1 204 self._current_task = None 205 self._unlock_builddir() 206 self.wait() 207 return 208 209 if already_fetched: 210 # This case is optimized to skip the fetch queue. 211 fetcher = None 212 self._fetch_exit(fetcher) 213 return 214 215 # Allow the Scheduler's fetch queue to control the 216 # number of concurrent fetchers. 217 fetcher.addExitListener(self._fetch_exit) 218 self._task_queued(fetcher) 219 self.scheduler.fetch.schedule(fetcher)
220
221 - def _fetch_exit(self, fetcher):
222 223 if fetcher is not None and \ 224 self._default_exit(fetcher) != os.EX_OK: 225 self._fetch_failed() 226 return 227 228 # discard successful fetch log 229 self._build_dir.clean_log() 230 pkg = self.pkg 231 logger = self.logger 232 opts = self.opts 233 pkg_count = self.pkg_count 234 scheduler = self.scheduler 235 settings = self.settings 236 features = settings.features 237 ebuild_path = self._ebuild_path 238 system_set = pkg.root_config.sets["system"] 239 240 #buildsyspkg: Check if we need to _force_ binary package creation 241 self._issyspkg = "buildsyspkg" in features and \ 242 system_set.findAtomForPackage(pkg) and \ 243 "buildpkg" not in features and \ 244 opts.buildpkg != 'n' 245 246 if ("buildpkg" in features or self._issyspkg) \ 247 and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): 248 249 self._buildpkg = True 250 251 msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ 252 (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) 253 short_msg = "emerge: (%s of %s) %s Compile" % \ 254 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 255 logger.log(msg, short_msg=short_msg) 256 257 else: 258 msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ 259 (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) 260 short_msg = "emerge: (%s of %s) %s Compile" % \ 261 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 262 logger.log(msg, short_msg=short_msg) 263 264 build = EbuildExecuter(background=self.background, pkg=pkg, 265 scheduler=scheduler, settings=settings) 266 self._start_task(build, self._build_exit)
267
268 - def _fetch_failed(self):
269 # We only call the pkg_nofetch phase if either RESTRICT=fetch 270 # is set or the package has explicitly overridden the default 271 # pkg_nofetch implementation. This allows specialized messages 272 # to be displayed for problematic packages even though they do 273 # not set RESTRICT=fetch (bug #336499). 274 275 if 'fetch' not in self.pkg.restrict and \ 276 'nofetch' not in self.pkg.defined_phases: 277 self._unlock_builddir() 278 self.wait() 279 return 280 281 self.returncode = None 282 nofetch_phase = EbuildPhase(background=self.background, 283 phase='nofetch', scheduler=self.scheduler, settings=self.settings) 284 self._start_task(nofetch_phase, self._nofetch_exit)
285
286 - def _nofetch_exit(self, nofetch_phase):
287 self._final_exit(nofetch_phase) 288 self._unlock_builddir() 289 self.returncode = 1 290 self.wait()
291
292 - def _unlock_builddir(self):
295
296 - def _build_exit(self, build):
297 if self._default_exit(build) != os.EX_OK: 298 self._unlock_builddir() 299 self.wait() 300 return 301 302 buildpkg = self._buildpkg 303 304 if not buildpkg: 305 self._final_exit(build) 306 self.wait() 307 return 308 309 if self._issyspkg: 310 msg = ">>> This is a system package, " + \ 311 "let's pack a rescue tarball.\n" 312 self.scheduler.output(msg, 313 log_path=self.settings.get("PORTAGE_LOG_FILE")) 314 315 binpkg_tasks = TaskSequence() 316 requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() 317 for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: 318 if pkg_fmt in requested_binpkg_formats: 319 if pkg_fmt == "rpm": 320 binpkg_tasks.add(EbuildPhase(background=self.background, 321 phase="rpm", scheduler=self.scheduler, 322 settings=self.settings)) 323 else: 324 task = EbuildBinpkg( 325 background=self.background, 326 pkg=self.pkg, scheduler=self.scheduler, 327 settings=self.settings) 328 binpkg_tasks.add(task) 329 # Guarantee that _record_binpkg_info is called 330 # immediately after EbuildBinpkg. Note that 331 # task.addExitListener does not provide the 332 # necessary guarantee (see bug 578204). 333 binpkg_tasks.add(self._RecordBinpkgInfo( 334 ebuild_binpkg=task, ebuild_build=self)) 335 336 if binpkg_tasks: 337 self._start_task(binpkg_tasks, self._buildpkg_exit) 338 return 339 340 self._final_exit(build) 341 self.wait()
342
343 - class _RecordBinpkgInfo(AsynchronousTask):
344 """ 345 This class wraps the EbuildBuild _record_binpkg_info method 346 with an AsynchronousTask interface, so that it can be 347 scheduled as a member of a TaskSequence. 348 """ 349 350 __slots__ = ('ebuild_binpkg', 'ebuild_build',) 351
352 - def _start(self):
355
356 - def _buildpkg_exit(self, packager):
357 """ 358 Released build dir lock when there is a failure or 359 when in buildpkgonly mode. Otherwise, the lock will 360 be released when merge() is called. 361 """ 362 363 if self._default_exit(packager) != os.EX_OK: 364 self._unlock_builddir() 365 self.wait() 366 return 367 368 if self.opts.buildpkgonly: 369 phase = 'success_hooks' 370 success_hooks = MiscFunctionsProcess( 371 background=self.background, 372 commands=[phase], phase=phase, 373 scheduler=self.scheduler, settings=self.settings) 374 self._start_task(success_hooks, 375 self._buildpkgonly_success_hook_exit) 376 return 377 378 # Continue holding the builddir lock until 379 # after the package has been installed. 380 self._current_task = None 381 self.returncode = packager.returncode 382 self.wait()
383
384 - def _record_binpkg_info(self, task):
385 if task.returncode != os.EX_OK: 386 return 387 388 # Save info about the created binary package, so that 389 # identifying information can be passed to the install 390 # task, to be recorded in the installed package database. 391 pkg = task.get_binpkg_info() 392 infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], 393 "build-info") 394 info = { 395 "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], 396 } 397 if pkg.build_id is not None: 398 info["BUILD_ID"] = "%s\n" % pkg.build_id 399 for k, v in info.items(): 400 with io.open(_unicode_encode(os.path.join(infoloc, k), 401 encoding=_encodings['fs'], errors='strict'), 402 mode='w', encoding=_encodings['repo.content'], 403 errors='strict') as f: 404 f.write(v)
405
406 - def _buildpkgonly_success_hook_exit(self, success_hooks):
407 self._default_exit(success_hooks) 408 self.returncode = None 409 # Need to call "clean" phase for buildpkgonly mode 410 portage.elog.elog_process(self.pkg.cpv, self.settings) 411 phase = 'clean' 412 clean_phase = EbuildPhase(background=self.background, 413 phase=phase, scheduler=self.scheduler, settings=self.settings) 414 self._start_task(clean_phase, self._clean_exit)
415
416 - def _clean_exit(self, clean_phase):
417 if self._final_exit(clean_phase) != os.EX_OK or \ 418 self.opts.buildpkgonly: 419 self._unlock_builddir() 420 self.wait()
421
422 - def create_install_task(self):
423 """ 424 Install the package and then clean up and release locks. 425 Only call this after the build has completed successfully 426 and neither fetchonly nor buildpkgonly mode are enabled. 427 """ 428 429 ldpath_mtimes = self.ldpath_mtimes 430 logger = self.logger 431 pkg = self.pkg 432 pkg_count = self.pkg_count 433 settings = self.settings 434 world_atom = self.world_atom 435 ebuild_path = self._ebuild_path 436 tree = self._tree 437 438 task = EbuildMerge(exit_hook=self._install_exit, 439 find_blockers=self.find_blockers, 440 ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, 441 pkg_count=pkg_count, pkg_path=ebuild_path, 442 scheduler=self.scheduler, 443 settings=settings, tree=tree, world_atom=world_atom) 444 445 msg = " === (%s of %s) Merging (%s::%s)" % \ 446 (pkg_count.curval, pkg_count.maxval, 447 pkg.cpv, ebuild_path) 448 short_msg = "emerge: (%s of %s) %s Merge" % \ 449 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 450 logger.log(msg, short_msg=short_msg) 451 452 return task
453
454 - def _install_exit(self, task):
455 self._unlock_builddir()
456