Package _emerge :: Module EbuildBuild
[hide private]

Source Code for Module _emerge.EbuildBuild

  1  # Copyright 1999-2014 Gentoo Foundation 
  2  # Distributed under the terms of the GNU General Public License v2 
  3   
  4  from __future__ import unicode_literals 
  5   
  6  import io 
  7   
  8  import _emerge.emergelog 
  9  from _emerge.AsynchronousTask import AsynchronousTask 
 10  from _emerge.EbuildExecuter import EbuildExecuter 
 11  from _emerge.EbuildPhase import EbuildPhase 
 12  from _emerge.EbuildBinpkg import EbuildBinpkg 
 13  from _emerge.EbuildFetcher import EbuildFetcher 
 14  from _emerge.CompositeTask import CompositeTask 
 15  from _emerge.EbuildMerge import EbuildMerge 
 16  from _emerge.EbuildFetchonly import EbuildFetchonly 
 17  from _emerge.EbuildBuildDir import EbuildBuildDir 
 18  from _emerge.MiscFunctionsProcess import MiscFunctionsProcess 
 19  from _emerge.TaskSequence import TaskSequence 
 20   
 21  import portage 
 22  from portage import _encodings, _unicode_decode, _unicode_encode, os 
 23  from portage.package.ebuild.digestcheck import digestcheck 
 24  from portage.package.ebuild.doebuild import _check_temp_dir 
 25  from portage.package.ebuild._spawn_nofetch import SpawnNofetchWithoutBuilddir 
 26   
27 -class EbuildBuild(CompositeTask):
28 29 __slots__ = ("args_set", "config_pool", "find_blockers", 30 "ldpath_mtimes", "logger", "opts", "pkg", "pkg_count", 31 "prefetcher", "settings", "world_atom") + \ 32 ("_build_dir", "_buildpkg", "_ebuild_path", "_issyspkg", "_tree") 33
34 - def _start(self):
35 36 pkg = self.pkg 37 settings = self.settings 38 39 if not self.opts.fetchonly: 40 rval = _check_temp_dir(settings) 41 if rval != os.EX_OK: 42 self.returncode = rval 43 self._current_task = None 44 self._async_wait() 45 return 46 47 root_config = pkg.root_config 48 tree = "porttree" 49 self._tree = tree 50 portdb = root_config.trees[tree].dbapi 51 settings.setcpv(pkg) 52 settings.configdict["pkg"]["EMERGE_FROM"] = "ebuild" 53 if self.opts.buildpkgonly: 54 settings.configdict["pkg"]["MERGE_TYPE"] = "buildonly" 55 else: 56 settings.configdict["pkg"]["MERGE_TYPE"] = "source" 57 ebuild_path = portdb.findname(pkg.cpv, myrepo=pkg.repo) 58 if ebuild_path is None: 59 raise AssertionError("ebuild not found for '%s'" % pkg.cpv) 60 self._ebuild_path = ebuild_path 61 portage.doebuild_environment(ebuild_path, 'setup', 62 settings=self.settings, db=portdb) 63 64 # Check the manifest here since with --keep-going mode it's 65 # currently possible to get this far with a broken manifest. 66 if not self._check_manifest(): 67 self.returncode = 1 68 self._current_task = None 69 self._async_wait() 70 return 71 72 prefetcher = self.prefetcher 73 if prefetcher is None: 74 pass 75 elif prefetcher.isAlive() and \ 76 prefetcher.poll() is None: 77 78 if not self.background: 79 fetch_log = os.path.join( 80 _emerge.emergelog._emerge_log_dir, 'emerge-fetch.log') 81 msg = ( 82 'Fetching files in the background.', 83 'To view fetch progress, run in another terminal:', 84 'tail -f %s' % fetch_log, 85 ) 86 out = portage.output.EOutput() 87 for l in msg: 88 out.einfo(l) 89 90 self._current_task = prefetcher 91 prefetcher.addExitListener(self._prefetch_exit) 92 return 93 94 self._prefetch_exit(prefetcher)
95
96 - def _check_manifest(self):
97 success = True 98 99 settings = self.settings 100 if 'strict' in settings.features and \ 101 'digest' not in settings.features: 102 settings['O'] = os.path.dirname(self._ebuild_path) 103 quiet_setting = settings.get('PORTAGE_QUIET') 104 settings['PORTAGE_QUIET'] = '1' 105 try: 106 success = digestcheck([], settings, strict=True) 107 finally: 108 if quiet_setting: 109 settings['PORTAGE_QUIET'] = quiet_setting 110 else: 111 del settings['PORTAGE_QUIET'] 112 113 return success
114
115 - def _prefetch_exit(self, prefetcher):
116 117 if self._was_cancelled(): 118 self.wait() 119 return 120 121 opts = self.opts 122 pkg = self.pkg 123 settings = self.settings 124 125 if opts.fetchonly: 126 if opts.pretend: 127 fetcher = EbuildFetchonly( 128 fetch_all=opts.fetch_all_uri, 129 pkg=pkg, pretend=opts.pretend, 130 settings=settings) 131 retval = fetcher.execute() 132 self.returncode = retval 133 self.wait() 134 return 135 else: 136 fetcher = EbuildFetcher( 137 config_pool=self.config_pool, 138 ebuild_path=self._ebuild_path, 139 fetchall=self.opts.fetch_all_uri, 140 fetchonly=self.opts.fetchonly, 141 background=False, 142 logfile=None, 143 pkg=self.pkg, 144 scheduler=self.scheduler) 145 self._start_task(fetcher, self._fetchonly_exit) 146 return 147 148 self._build_dir = EbuildBuildDir( 149 scheduler=self.scheduler, settings=settings) 150 self._build_dir.lock() 151 152 # Cleaning needs to happen before fetch, since the build dir 153 # is used for log handling. 154 msg = " === (%s of %s) Cleaning (%s::%s)" % \ 155 (self.pkg_count.curval, self.pkg_count.maxval, 156 self.pkg.cpv, self._ebuild_path) 157 short_msg = "emerge: (%s of %s) %s Clean" % \ 158 (self.pkg_count.curval, self.pkg_count.maxval, self.pkg.cpv) 159 self.logger.log(msg, short_msg=short_msg) 160 161 pre_clean_phase = EbuildPhase(background=self.background, 162 phase='clean', scheduler=self.scheduler, settings=self.settings) 163 self._start_task(pre_clean_phase, self._pre_clean_exit)
164
165 - def _fetchonly_exit(self, fetcher):
166 self._final_exit(fetcher) 167 if self.returncode != os.EX_OK: 168 self.returncode = None 169 portdb = self.pkg.root_config.trees[self._tree].dbapi 170 self._start_task(SpawnNofetchWithoutBuilddir( 171 background=self.background, 172 portdb=portdb, 173 ebuild_path=self._ebuild_path, 174 scheduler=self.scheduler, 175 settings=self.settings), 176 self._nofetch_without_builddir_exit) 177 return 178 179 self.wait()
180
181 - def _nofetch_without_builddir_exit(self, nofetch):
182 self._final_exit(nofetch) 183 self.returncode = 1 184 self.wait()
185
186 - def _pre_clean_exit(self, pre_clean_phase):
187 if self._default_exit(pre_clean_phase) != os.EX_OK: 188 self._unlock_builddir() 189 self.wait() 190 return 191 192 # for log handling 193 portage.prepare_build_dirs(self.pkg.root, self.settings, 1) 194 195 fetcher = EbuildFetcher(config_pool=self.config_pool, 196 ebuild_path=self._ebuild_path, 197 fetchall=self.opts.fetch_all_uri, 198 fetchonly=self.opts.fetchonly, 199 background=self.background, 200 logfile=self.settings.get('PORTAGE_LOG_FILE'), 201 pkg=self.pkg, scheduler=self.scheduler) 202 203 try: 204 already_fetched = fetcher.already_fetched(self.settings) 205 except portage.exception.InvalidDependString as e: 206 msg_lines = [] 207 msg = "Fetch failed for '%s' due to invalid SRC_URI: %s" % \ 208 (self.pkg.cpv, e) 209 msg_lines.append(msg) 210 fetcher._eerror(msg_lines) 211 portage.elog.elog_process(self.pkg.cpv, self.settings) 212 self.returncode = 1 213 self._current_task = None 214 self._unlock_builddir() 215 self.wait() 216 return 217 218 if already_fetched: 219 # This case is optimized to skip the fetch queue. 220 fetcher = None 221 self._fetch_exit(fetcher) 222 return 223 224 # Allow the Scheduler's fetch queue to control the 225 # number of concurrent fetchers. 226 fetcher.addExitListener(self._fetch_exit) 227 self._task_queued(fetcher) 228 self.scheduler.fetch.schedule(fetcher)
229
230 - def _fetch_exit(self, fetcher):
231 232 if fetcher is not None and \ 233 self._default_exit(fetcher) != os.EX_OK: 234 self._fetch_failed() 235 return 236 237 # discard successful fetch log 238 self._build_dir.clean_log() 239 pkg = self.pkg 240 logger = self.logger 241 opts = self.opts 242 pkg_count = self.pkg_count 243 scheduler = self.scheduler 244 settings = self.settings 245 features = settings.features 246 ebuild_path = self._ebuild_path 247 system_set = pkg.root_config.sets["system"] 248 249 #buildsyspkg: Check if we need to _force_ binary package creation 250 self._issyspkg = "buildsyspkg" in features and \ 251 system_set.findAtomForPackage(pkg) and \ 252 "buildpkg" not in features and \ 253 opts.buildpkg != 'n' 254 255 if ("buildpkg" in features or self._issyspkg) \ 256 and not self.opts.buildpkg_exclude.findAtomForPackage(pkg): 257 258 self._buildpkg = True 259 260 msg = " === (%s of %s) Compiling/Packaging (%s::%s)" % \ 261 (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) 262 short_msg = "emerge: (%s of %s) %s Compile" % \ 263 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 264 logger.log(msg, short_msg=short_msg) 265 266 else: 267 msg = " === (%s of %s) Compiling/Merging (%s::%s)" % \ 268 (pkg_count.curval, pkg_count.maxval, pkg.cpv, ebuild_path) 269 short_msg = "emerge: (%s of %s) %s Compile" % \ 270 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 271 logger.log(msg, short_msg=short_msg) 272 273 build = EbuildExecuter(background=self.background, pkg=pkg, 274 scheduler=scheduler, settings=settings) 275 self._start_task(build, self._build_exit)
276
277 - def _fetch_failed(self):
278 # We only call the pkg_nofetch phase if either RESTRICT=fetch 279 # is set or the package has explicitly overridden the default 280 # pkg_nofetch implementation. This allows specialized messages 281 # to be displayed for problematic packages even though they do 282 # not set RESTRICT=fetch (bug #336499). 283 284 if 'fetch' not in self.pkg.restrict and \ 285 'nofetch' not in self.pkg.defined_phases: 286 self._unlock_builddir() 287 self.wait() 288 return 289 290 self.returncode = None 291 nofetch_phase = EbuildPhase(background=self.background, 292 phase='nofetch', scheduler=self.scheduler, settings=self.settings) 293 self._start_task(nofetch_phase, self._nofetch_exit)
294
295 - def _nofetch_exit(self, nofetch_phase):
296 self._final_exit(nofetch_phase) 297 self._unlock_builddir() 298 self.returncode = 1 299 self.wait()
300
301 - def _unlock_builddir(self):
304
305 - def _build_exit(self, build):
306 if self._default_exit(build) != os.EX_OK: 307 self._unlock_builddir() 308 self.wait() 309 return 310 311 buildpkg = self._buildpkg 312 313 if not buildpkg: 314 self._final_exit(build) 315 self.wait() 316 return 317 318 if self._issyspkg: 319 msg = ">>> This is a system package, " + \ 320 "let's pack a rescue tarball.\n" 321 self.scheduler.output(msg, 322 log_path=self.settings.get("PORTAGE_LOG_FILE")) 323 324 binpkg_tasks = TaskSequence() 325 requested_binpkg_formats = self.settings.get("PORTAGE_BINPKG_FORMAT", "tar").split() 326 for pkg_fmt in portage.const.SUPPORTED_BINPKG_FORMATS: 327 if pkg_fmt in requested_binpkg_formats: 328 if pkg_fmt == "rpm": 329 binpkg_tasks.add(EbuildPhase(background=self.background, 330 phase="rpm", scheduler=self.scheduler, 331 settings=self.settings)) 332 else: 333 task = EbuildBinpkg( 334 background=self.background, 335 pkg=self.pkg, scheduler=self.scheduler, 336 settings=self.settings) 337 binpkg_tasks.add(task) 338 # Guarantee that _record_binpkg_info is called 339 # immediately after EbuildBinpkg. Note that 340 # task.addExitListener does not provide the 341 # necessary guarantee (see bug 578204). 342 binpkg_tasks.add(self._RecordBinpkgInfo( 343 ebuild_binpkg=task, ebuild_build=self)) 344 345 if binpkg_tasks: 346 self._start_task(binpkg_tasks, self._buildpkg_exit) 347 return 348 349 self._final_exit(build) 350 self.wait()
351
352 - class _RecordBinpkgInfo(AsynchronousTask):
353 """ 354 This class wraps the EbuildBuild _record_binpkg_info method 355 with an AsynchronousTask interface, so that it can be 356 scheduled as a member of a TaskSequence. 357 """ 358 359 __slots__ = ('ebuild_binpkg', 'ebuild_build',) 360
361 - def _start(self):
364
365 - def _buildpkg_exit(self, packager):
366 """ 367 Released build dir lock when there is a failure or 368 when in buildpkgonly mode. Otherwise, the lock will 369 be released when merge() is called. 370 """ 371 372 if self._default_exit(packager) != os.EX_OK: 373 self._unlock_builddir() 374 self.wait() 375 return 376 377 if self.opts.buildpkgonly: 378 phase = 'success_hooks' 379 success_hooks = MiscFunctionsProcess( 380 background=self.background, 381 commands=[phase], phase=phase, 382 scheduler=self.scheduler, settings=self.settings) 383 self._start_task(success_hooks, 384 self._buildpkgonly_success_hook_exit) 385 return 386 387 # Continue holding the builddir lock until 388 # after the package has been installed. 389 self._current_task = None 390 self.returncode = packager.returncode 391 self.wait()
392
393 - def _record_binpkg_info(self, task):
394 if task.returncode != os.EX_OK: 395 return 396 397 # Save info about the created binary package, so that 398 # identifying information can be passed to the install 399 # task, to be recorded in the installed package database. 400 pkg = task.get_binpkg_info() 401 infoloc = os.path.join(self.settings["PORTAGE_BUILDDIR"], 402 "build-info") 403 info = { 404 "BINPKGMD5": "%s\n" % pkg._metadata["MD5"], 405 } 406 if pkg.build_id is not None: 407 info["BUILD_ID"] = "%s\n" % pkg.build_id 408 for k, v in info.items(): 409 with io.open(_unicode_encode(os.path.join(infoloc, k), 410 encoding=_encodings['fs'], errors='strict'), 411 mode='w', encoding=_encodings['repo.content'], 412 errors='strict') as f: 413 f.write(v)
414
415 - def _buildpkgonly_success_hook_exit(self, success_hooks):
416 self._default_exit(success_hooks) 417 self.returncode = None 418 # Need to call "clean" phase for buildpkgonly mode 419 portage.elog.elog_process(self.pkg.cpv, self.settings) 420 phase = 'clean' 421 clean_phase = EbuildPhase(background=self.background, 422 phase=phase, scheduler=self.scheduler, settings=self.settings) 423 self._start_task(clean_phase, self._clean_exit)
424
425 - def _clean_exit(self, clean_phase):
426 if self._final_exit(clean_phase) != os.EX_OK or \ 427 self.opts.buildpkgonly: 428 self._unlock_builddir() 429 self.wait()
430
431 - def create_install_task(self):
432 """ 433 Install the package and then clean up and release locks. 434 Only call this after the build has completed successfully 435 and neither fetchonly nor buildpkgonly mode are enabled. 436 """ 437 438 ldpath_mtimes = self.ldpath_mtimes 439 logger = self.logger 440 pkg = self.pkg 441 pkg_count = self.pkg_count 442 settings = self.settings 443 world_atom = self.world_atom 444 ebuild_path = self._ebuild_path 445 tree = self._tree 446 447 task = EbuildMerge(exit_hook=self._install_exit, 448 find_blockers=self.find_blockers, 449 ldpath_mtimes=ldpath_mtimes, logger=logger, pkg=pkg, 450 pkg_count=pkg_count, pkg_path=ebuild_path, 451 scheduler=self.scheduler, 452 settings=settings, tree=tree, world_atom=world_atom) 453 454 msg = " === (%s of %s) Merging (%s::%s)" % \ 455 (pkg_count.curval, pkg_count.maxval, 456 pkg.cpv, ebuild_path) 457 short_msg = "emerge: (%s of %s) %s Merge" % \ 458 (pkg_count.curval, pkg_count.maxval, pkg.cpv) 459 logger.log(msg, short_msg=short_msg) 460 461 return task
462
463 - def _install_exit(self, task):
464 self._unlock_builddir()
465