Traceback (most recent call last):
  File "/usr/bin/pungi-koji", line 326, in <module>
    main()
  File "/usr/bin/pungi-koji", line 180, in main
    run_compose(compose)
  File "/usr/bin/pungi-koji", line 240, in run_compose
    gather_phase.start()
  File "/usr/lib/python2.7/site-packages/pungi/phases/base.py", line 62, in start
    self.run()
  File "/usr/lib/python2.7/site-packages/pungi/phases/gather/__init__.py", line 123, in run
    link_files(self.compose, arch, variant, pkg_map[arch][variant.uid], self.pkgset_phase.package_sets, manifest=manifest)
  File "/usr/lib/python2.7/site-packages/pungi/phases/gather/link.py", line 101, in link_files
    pool.stop()
  File "/usr/lib/python2.7/site-packages/kobo/threads.py", line 65, in run
    self.process(item, num)
  File "/usr/lib/python2.7/site-packages/pungi/linker.py", line 43, in process
    self.pool.linker.link(src, dst, link_type=self.pool.link_type)
  File "/usr/lib/python2.7/site-packages/pungi/linker.py", line 248, in link
    self._link_file(src, dst, link_type)
  File "/usr/lib/python2.7/site-packages/pungi/linker.py", line 239, in _link_file
    self.hardlink(src, dst)
  File "/usr/lib/python2.7/site-packages/pungi/linker.py", line 129, in hardlink
    os.link(src, dst)
OSError: [Errno 18] Invalid cross-device link
Frame hardlink in /usr/lib/python2.7/site-packages/pungi/linker.py at line 129
<CODE>
     122         msg = "Hardlinking %s to %s" % (src, dst)
     123         if self.test:
     124             self.log_info("TEST: %s" % msg)
     125             return
     126         self.log_info(msg)
     127 
     128         try:
-->  129             os.link(src, dst)
     130         except OSError as ex:
     131             if ex.errno != errno.EEXIST:
     132                 raise
     133             if self._is_same(src, dst):
     134                 if not self._is_same_type(src, dst):
     135                     self.log_error("File %s already exists but has different type than %s" % (dst, src))
</CODE>
<LOCALS>
                 dst = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/source/tree/Packages/ConsoleKit-0.4.5-9.fc22.src.rpm'
                  ex = OSError(18, 'Invalid cross-device link')
                 msg = 'Hardlinking /mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/src/ConsoleKit-0.4.5-9.fc22.src.rpm to /mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/source/tree/Packages/ConsoleKit-0.4.5-
                self = <pungi.linker.Linker object at 0x7f8cc454cf50>
     self._inode_map = {}
        self._logger = None
      self._precache = {}
    self.always_copy = []
self.ignore_existing = False
           self.test = False
                 src = '/mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/src/ConsoleKit-0.4.5-9.fc22.src.rpm'
</LOCALS>
Frame _link_file in /usr/lib/python2.7/site-packages/pungi/linker.py at line 239
<CODE>
     232                 relative = link_type != "abspath-symlink"
     233                 self.symlink(src, dst, relative)
     234         elif link_type == "hardlink-or-copy":
     235             if not self.hardlink_on_dest(src, dst):
     236                 src_stat = os.stat(src)
     237                 dst_stat = os.stat(os.path.dirname(dst))
     238                 if src_stat.st_dev == dst_stat.st_dev:
-->  239                     self.hardlink(src, dst)
     240                 else:
     241                     self.copy(src, dst)
     242         else:
     243             raise ValueError("Unknown link_type: %s" % link_type)
     244 
     245     def link(self, src, dst, link_type="hardlink-or-copy", scan=True):
</CODE>
<LOCALS>
                 dst = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/source/tree/Packages/ConsoleKit-0.4.5-9.fc22.src.rpm'
            dst_stat = posix.stat_result(st_mode=17917, st_ino=123996953, st_dev=38, st_nlink=2, st_uid=0, st_gid=101737, st_size=4096, st_atime=1434126741, st_mtime=1434126741, st_ctime=1434126741)
           link_type = 'hardlink-or-copy'
                self = <pungi.linker.Linker object at 0x7f8cc454cf50>
     self._inode_map = {}
        self._logger = None
      self._precache = {}
    self.always_copy = []
self.ignore_existing = False
           self.test = False
                 src = '/mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/src/ConsoleKit-0.4.5-9.fc22.src.rpm'
            src_stat = posix.stat_result(st_mode=33188, st_ino=99662109, st_dev=38, st_nlink=1, st_uid=48, st_gid=48, st_size=454419, st_atime=1434121866, st_mtime=1408127921, st_ctime=1408127934)
</LOCALS>
Frame link in /usr/lib/python2.7/site-packages/pungi/linker.py at line 248
<CODE>
     241                     self.copy(src, dst)
     242         else:
     243             raise ValueError("Unknown link_type: %s" % link_type)
     244 
     245     def link(self, src, dst, link_type="hardlink-or-copy", scan=True):
     246         """Link directories recursively."""
     247         if os.path.isfile(src) or os.path.islink(src):
-->  248             self._link_file(src, dst, link_type)
     249             return
     250 
     251         if os.path.isfile(dst):
     252             raise OSError(errno.EEXIST, "File exists")
     253 
     254         if not self.test:
</CODE>
<LOCALS>
                 dst = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/source/tree/Packages/ConsoleKit-0.4.5-9.fc22.src.rpm'
           link_type = 'hardlink-or-copy'
                scan = True
                self = <pungi.linker.Linker object at 0x7f8cc454cf50>
     self._inode_map = {}
        self._logger = None
      self._precache = {}
    self.always_copy = []
self.ignore_existing = False
           self.test = False
                 src = '/mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/src/ConsoleKit-0.4.5-9.fc22.src.rpm'
</LOCALS>
Frame process in /usr/lib/python2.7/site-packages/pungi/linker.py at line 43
<CODE>
      36 class LinkerThread(WorkerThread):
      37     def process(self, item, num):
      38         src, dst = item
      39 
      40         if (num % 100 == 0) or (num == self.pool.queue_total):
      41             self.pool.log_debug("Linked %s out of %s packages" % (num, self.pool.queue_total))
      42 
-->   43         self.pool.linker.link(src, dst, link_type=self.pool.link_type)
      44 
      45 
      46 class Linker(kobo.log.LoggingBase):
      47     def __init__(self, ignore_existing=False, always_copy=None, test=False, logger=None):
      48         kobo.log.LoggingBase.__init__(self, logger=logger)
      49         self.ignore_existing = ignore_existing
</CODE>
<LOCALS>
                 dst = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/source/tree/Packages/ConsoleKit-0.4.5-9.fc22.src.rpm'
                item = <ERROR WHILE CONVERTING VALUE TO STRING>
                 num = 2
                self = <LinkerThread(Thread-22, stopped 140242006824704)>
  self._Thread__args = <ERROR WHILE CONVERTING VALUE TO STRING>
 self._Thread__block = <Condition(<thread.lock object at 0x7f8c82838d10>, 0)>
self._Thread__daemonic = False
 self._Thread__ident = 140242006824704
self._Thread__initialized = True
self._Thread__kwargs = {}
  self._Thread__name = 'Thread-22'
self._Thread__started = <threading._Event object at 0x7f8cbb2982d0>
self._Thread__stderr = <open file '<stderr>', mode 'w' at 0x7f8cdab741e0>
self._Thread__stopped = True
self._Thread__target = None
self._Verbose__verbose = False
         self._block = <Condition(<thread.lock object at 0x7f8c82838d10>, 0)>
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140242006824704
           self.kill = True
           self.name = 'Thread-22'
           self.pool = <pungi.linker.LinkerPool object at 0x7f8cc799ac90>
        self.running = False
                 src = '/mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/src/ConsoleKit-0.4.5-9.fc22.src.rpm'
</LOCALS>
Frame run in /usr/lib/python2.7/site-packages/kobo/threads.py at line 65
<CODE>
      58 
      59             self.pool.queue_get_lock.acquire()
      60             self.pool.queue_processed += 1
      61             num = self.pool.queue_processed
      62             self.pool.queue_get_lock.release()
      63 
      64             try:
-->   65                 self.process(item, num)
      66             except:
      67                 self.failed = True
      68                 self.pool.exceptions.append(sys.exc_info())
      69                 self.pool.kill()
      70 
      71     def process(self, item, num):
</CODE>
<LOCALS>
                item = <ERROR WHILE CONVERTING VALUE TO STRING>
                 num = 2
                self = <LinkerThread(Thread-22, stopped 140242006824704)>
  self._Thread__args = <ERROR WHILE CONVERTING VALUE TO STRING>
 self._Thread__block = <Condition(<thread.lock object at 0x7f8c82838d10>, 0)>
self._Thread__daemonic = False
 self._Thread__ident = 140242006824704
self._Thread__initialized = True
self._Thread__kwargs = {}
  self._Thread__name = 'Thread-22'
self._Thread__started = <threading._Event object at 0x7f8cbb2982d0>
self._Thread__stderr = <open file '<stderr>', mode 'w' at 0x7f8cdab741e0>
self._Thread__stopped = True
self._Thread__target = None
self._Verbose__verbose = False
         self._block = <Condition(<thread.lock object at 0x7f8c82838d10>, 0)>
         self.daemon = False
         self.failed = True
    self.get_timeout = 1
          self.ident = 140242006824704
           self.kill = True
           self.name = 'Thread-22'
           self.pool = <pungi.linker.LinkerPool object at 0x7f8cc799ac90>
        self.running = False
</LOCALS>
Frame link_files in /usr/lib/python2.7/site-packages/pungi/phases/gather/link.py at line 101
<CODE>
      94         # update rpm manifest
      95         pkg_obj = pkg_set[pkg["path"]]
      96         nevra = pkg_obj.nevra
      97         src_nevra = _get_src_nevra(compose, pkg_obj, srpm_map)
      98         manifest.add(variant.uid, arch, nevra, path=dst_relpath, sigkey=pkg_obj.signature, category="debug", srpm_nevra=src_nevra)
      99 
     100     pool.start()
-->  101     pool.stop()
     102     compose.log_info("[DONE ] %s" % msg)
</CODE>
<LOCALS>
                arch = 'armhfp'
             compose = <pungi.compose.Compose object at 0x7f8cc79b8c90>
                 dst = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/armhfp/debug/tree/Packages/zziplib-debuginfo-0.13.62-5.fc22.armv7hl.rpm'
         dst_relpath = 'Cloud/armhfp/debug/tree/Packages/zziplib-debuginfo-0.13.62-5.fc22.armv7hl.rpm'
                   i = 9
           link_type = 'hardlink-or-copy'
            manifest = <productmd.rpms.Rpms object at 0x7f8c8624dfd0>
                 msg = 'Linking packages (arch: armhfp, variant: Cloud)'
               nevra = 'zziplib-debuginfo-0:0.13.62-5.fc22.armv7hl'
        packages_dir = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/Cloud/armhfp/debug/tree/Packages'
packages_dir_relpath = 'Cloud/armhfp/debug/tree/Packages'
                 pkg = {'path': '/mnt/koji/packages/zziplib/0.13.62/5.fc22/armv7hl/zziplib-debuginfo-0.13.62-5.fc22.armv7hl.rpm', 'flags': []}
             pkg_map = {'debuginfo': [{'path': '/mnt/koji/packages/Canna/3.7p3/44.fc22/armv7hl/Canna-debuginfo-3.7p3-44.fc22.armv7hl.rpm', 'flags': []}, {'path': '/mnt/koji/packages/ConsoleKit/0.4.5/9.fc22/armv7hl/ConsoleKi
             pkg_obj = zziplib-debuginfo-0.13.62-5.fc22.armv7hl.rpm
             pkg_set = <pungi.phases.pkgset.pkgsets.PackageSetBase object at 0x7f8c8bf31f50>
            pkg_sets = {'x86_64': <pungi.phases.pkgset.pkgsets.PackageSetBase object at 0x7f8ca179c490>, 'i386': <pungi.phases.pkgset.pkgsets.PackageSetBase object at 0x7f8cc459c410>, 'global': <pungi.phases.pkgset.pkgsets.
                pool = <pungi.linker.LinkerPool object at 0x7f8cc799ac90>
           src_nevra = 'zziplib-0:0.13.62-5.fc22.src'
            srpm_map = {'hunspell-br-0.8-6.fc21.src.rpm': 'hunspell-br-1:0.8-6.fc21.src', 'linux-atm-2.5.1-12.fc23.src.rpm': 'linux-atm-0:2.5.1-12.fc23.src', 'perl-HTTP-Tiny-0.056-2.fc23.src.rpm': 'perl-HTTP-Tiny-0:0.056-2.
             variant = <pungi.wrappers.variants.Variant object at 0x7f8cc45acb50>
</LOCALS>
Frame run in /usr/lib/python2.7/site-packages/pungi/phases/gather/__init__.py at line 123
<CODE>
     116         manifest.compose.id = self.compose.compose_id
     117         manifest.compose.type = self.compose.compose_type
     118         manifest.compose.date = self.compose.compose_date
     119         manifest.compose.respin = self.compose.compose_respin
     120 
     121         for arch in self.compose.get_arches():
     122             for variant in self.compose.get_variants(arch=arch):
-->  123                 link_files(self.compose, arch, variant, pkg_map[arch][variant.uid], self.pkgset_phase.package_sets, manifest=manifest)
     124 
     125         self.compose.log_info("Writing RPM manifest: %s" % manifest_file)
     126         manifest.dump(manifest_file)
     127 
     128 
     129 def get_parent_pkgs(arch, variant, result_dict):
</CODE>
<LOCALS>
                arch = 'armhfp'
            manifest = <productmd.rpms.Rpms object at 0x7f8c8624dfd0>
       manifest_file = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/compose/metadata/rpms.json'
             pkg_map = {'x86_64': {'Workstation': {'debuginfo': [{'path': '/mnt/koji/packages/Canna/3.7p3/44.fc22/x86_64/Canna-debuginfo-3.7p3-44.fc22.x86_64.rpm', 'flags': []}, {'path': '/mnt/koji/packages/ConsoleKit/0.4.5
                self = <pungi.phases.gather.GatherPhase object at 0x7f8cc454cbd0>
       self._skipped = False
        self.compose = <pungi.compose.Compose object at 0x7f8cc79b8c90>
 self.config_options = <ERROR WHILE CONVERTING VALUE TO STRING>
       self.finished = False
            self.msg = '---------- PHASE: GATHER ----------'
           self.name = 'gather'
   self.pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8cc454c2d0>
             variant = <pungi.wrappers.variants.Variant object at 0x7f8cc45acb50>
</LOCALS>
Frame start in /usr/lib/python2.7/site-packages/pungi/phases/base.py at line 62
<CODE>
      55     def start(self):
      56         self._skipped = self.skip()
      57         if self._skipped:
      58             self.compose.log_warning("[SKIP ] %s" % self.msg)
      59             self.finished = True
      60             return
      61         self.compose.log_info("[BEGIN] %s" % self.msg)
-->   62         self.run()
      63 
      64     def stop(self):
      65         if self.finished:
      66             return
      67         if hasattr(self, "pool"):
      68             self.pool.stop()
</CODE>
<LOCALS>
                self = <pungi.phases.gather.GatherPhase object at 0x7f8cc454cbd0>
       self._skipped = False
        self.compose = <pungi.compose.Compose object at 0x7f8cc79b8c90>
 self.config_options = <ERROR WHILE CONVERTING VALUE TO STRING>
       self.finished = False
            self.msg = '---------- PHASE: GATHER ----------'
           self.name = 'gather'
   self.pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8cc454c2d0>
</LOCALS>
Frame run_compose in /usr/bin/pungi-koji at line 240
<CODE>
     233     pkgset_phase.start()
     234     pkgset_phase.stop()
     235 
     236     # BUILDINSTALL phase - start
     237     buildinstall_phase.start()
     238 
     239     # GATHER phase
-->  240     gather_phase.start()
     241     gather_phase.stop()
     242 
     243     # EXTRA_FILES phase
     244     extrafiles_phase.start()
     245     extrafiles_phase.stop()
     246 
</CODE>
<LOCALS>
                 arg = '--no-label'
  buildinstall_phase = <pungi.phases.buildinstall.BuildinstallPhase object at 0x7f8cc454ca10>
             compose = <pungi.compose.Compose object at 0x7f8cc79b8c90>
         config_dump = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/logs/global/config-dump_2015-06-12_14-54-59.global.log'
     createiso_phase = <pungi.phases.createiso.CreateisoPhase object at 0x7f8cc454cc90>
    createrepo_phase = <pungi.phases.createrepo.CreaterepoPhase object at 0x7f8cc454c750>
            date_str = '2015-06-12_14-54-59'
              errors = []
    extrafiles_phase = <pungi.phases.extra_files.ExtraFilesPhase object at 0x7f8cc454cb10>
        gather_phase = <pungi.phases.gather.GatherPhase object at 0x7f8cc454cbd0>
          init_phase = <pungi.phases.init.InitPhase object at 0x7f8cc45acb10>
    liveimages_phase = <pungi.phases.live_images.LiveImagesPhase object at 0x7f8cc454c9d0>
               phase = <pungi.phases.test.TestPhase object at 0x7f8cc454cd50>
        pkgset_phase = <pungi.phases.pkgset.PkgsetPhase object at 0x7f8cc454c2d0>
    productimg_phase = <pungi.phases.product_img.ProductimgPhase object at 0x7f8cc454c990>
               pungi = <module 'pungi' from '/usr/lib/python2.7/site-packages/pungi/__init__.pyc'>
          test_phase = <pungi.phases.test.TestPhase object at 0x7f8cc454cd50>
</LOCALS>
Frame main in /usr/bin/pungi-koji at line 180
<CODE>
     173     else:
     174         compose_dir = opts.compose_dir
     175 
     176     compose = Compose(conf, topdir=compose_dir, debug=opts.debug_mode, skip_phases=opts.skip_phase, just_phases=opts.just_phase,
     177                       old_composes=opts.old_composes, koji_event=opts.koji_event, supported=opts.supported, logger=logger)
     178     kobo.log.add_file_logger(logger, compose.paths.log.log_file("global", "pungi.log"))
     179     COMPOSE = compose
-->  180     run_compose(compose)
     181 
     182 
     183 def run_compose(compose):
     184     import pungi.phases
     185     import pungi.metadata
     186 
</CODE>
<LOCALS>
             Compose = <class 'pungi.compose.Compose'>
                args = []
             compose = <pungi.compose.Compose object at 0x7f8cc79b8c90>
         compose_dir = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2'
        compose_type = 'nightly'
                conf = {'pkgset_koji_path_prefix': '/mnt/koji', 'pkgset_koji_url': 'http://koji.fedoraproject.org/kojihub', 'additional_packages': [('.*', {'*': ['kernel.*', 'dracut.*', 'autocorr-.*', 'eclipse-nls-.*', 'hun
                kobo = <module 'kobo' from '/usr/lib/python2.7/site-packages/kobo/__init__.pyc'>
              logger = <logging.Logger object at 0x7f8cc799afd0>
                opts = <Values at 0x7f8cda8467e8: {'old_composes': [], 'compose_dir': None, 'version': None, 'koji_event': None, 'just_phase': [], 'target_dir': '/mnt/fedora_koji/compose/test', 'supported': False, 'label': 
              parser = <optparse.OptionParser instance at 0x7f8cda8902d8>
           productmd = <module 'productmd' from '/usr/lib/python2.7/site-packages/productmd/__init__.pyc'>
               pungi = <module 'pungi' from '/usr/lib/python2.7/site-packages/pungi/__init__.pyc'>
</LOCALS>
Frame <module> in /usr/bin/pungi-koji at line 326
<CODE>
     319 
     320     compose.log_info("Compose finished: %s" % compose.topdir)
     321     compose.write_status("FINISHED")
     322 
     323 
     324 if __name__ == "__main__":
     325     try:
-->  326         main()
     327     except (Exception, KeyboardInterrupt) as ex:
     328         if COMPOSE:
     329             tb_path = COMPOSE.paths.log.log_file("global", "traceback")
     330             COMPOSE.log_error("Exception: %s" % ex)
     331             COMPOSE.log_error("Extended traceback in: %s" % tb_path)
     332             COMPOSE.log_critical("Compose failed: %s" % COMPOSE.topdir)
</CODE>
<LOCALS>
             COMPOSE = <pungi.compose.Compose object at 0x7f8cc79b8c90>
        __builtins__ = <module '__builtin__' (built-in)>
             __doc__ = None
            __file__ = '/usr/bin/pungi-koji'
            __name__ = '__main__'
         __package__ = None
         __version__ = '4.0'
            datetime = <module 'datetime' from '/usr/lib64/python2.7/lib-dynload/datetime.so'>
                  ex = OSError(18, 'Invalid cross-device link')
             getpass = <module 'getpass' from '/usr/lib64/python2.7/getpass.pyc'>
                here = '/usr/bin'
                json = <module 'json' from '/usr/lib64/python2.7/json/__init__.pyc'>
                kobo = <module 'kobo' from '/usr/lib/python2.7/site-packages/kobo/__init__.pyc'>
              locale = <module 'locale' from '/usr/lib64/python2.7/locale.pyc'>
             logging = <module 'logging' from '/usr/lib64/python2.7/logging/__init__.pyc'>
                main = <function main at 0x7f8cda841488>
            optparse = <module 'optparse' from '/usr/lib64/python2.7/optparse.pyc'>
                  os = <module 'os' from '/usr/lib64/python2.7/os.pyc'>
               pipes = <module 'pipes' from '/usr/lib64/python2.7/pipes.pyc'>
         run_compose = <function run_compose at 0x7f8cda841500>
              socket = <module 'socket' from '/usr/lib64/python2.7/socket.pyc'>
                 sys = <module 'sys' (built-in)>
             tb_path = '/mnt/fedora_koji/compose/test/Fedora-23-20150612.n.2/logs/global/traceback.global.log'
</LOCALS>