cosaves.py - beta

This API was starting to become inadequate for Wrye Bash's current
needs. Most importantly:
 1. It was difficult to understand and maintain.
 2. It did not support enough of the cosave format.
 3. Pluggy support was all over the place.

This commit looks like a complete rewrite, but it was actually a
step-by-step refactoring comprising 100+ commits that got squashed down
to this final commit, since almost every step of the way would have
broken dev.

For reference:
  cosaves.py - alpha: 0a300af.

New features included in this commit are:

1. Add button for dumping Pluggy cosaves

  With the new cosaves API, we now have the capability to dump Pluggy
  cosaves as well, so this commit adds a button to do just that.

2. Correctly display save masters in SSE

  With SKSE 2.0.15, which adds the PLGN chunk, we can now display save
  master lists in SSE in the correct order (with ESLs and regular plugins
  interweaved).

Author: Infernio

Mopy/bash/basher/__init__.py

@@ -372,7 +372,7 @@ def SetFileInfo(self,fileInfo):
         if not fileInfo:
             return
         #--Fill data and populate
-        for mi, masters_name in enumerate(fileInfo.header.masters):
+        for mi, masters_name in enumerate(fileInfo.get_masters()):
             masterInfo = bosh.MasterInfo(masters_name)
             self.data_store[mi] = masterInfo
         self._reList()

Mopy/bash/basher/links.py

@@ -650,6 +650,7 @@ def InitSaveLinks():
     if bush.game.ess.canEditMore:
         SaveList.itemMenu.append(Save_Stats())
     SaveList.itemMenu.append(Save_StatObse())
+    SaveList.itemMenu.append(Save_StatPluggy())
     if bush.game.ess.canEditMore:
         #--------------------------------------------
         SaveList.itemMenu.append(SeparatorLink())

Mopy/bash/basher/saves_links.py

@@ -43,7 +43,7 @@
            'Save_EditCreatedEnchantmentCosts', 'Save_ImportFace',
            'Save_EditCreated', 'Save_ReweighPotions', 'Save_UpdateNPCLevels',
            'Save_ExportScreenshot', 'Save_Unbloat', 'Save_RepairAbomb',
-           'Save_RepairHair']
+           'Save_RepairHair', 'Save_StatPluggy']
 
 #------------------------------------------------------------------------------
 # Saves Links -----------------------------------------------------------------
@@ -758,25 +758,51 @@ def Execute(self):
 #------------------------------------------------------------------------------
 class Save_StatObse(AppendableLink, OneItemLink):
     """Dump .obse records."""
-    _text = _(u'%s Statistics') % bush.game.se.cosave_ext.lower()
-    _help = _(u'Dump %s records') % bush.game.se.cosave_ext.lower()
+    _text = _(u'Dump %s Contents') % bush.game.se.cosave_ext.lower()
+    _help = _(u'Dumps contents of associated %s cosave into a log.') % \
+            bush.game.se.se_abbrev
 
     def _append(self, window): return bool(bush.game.se.se_abbrev)
 
     def _enable(self):
         if not super(Save_StatObse, self)._enable(): return False
-        cosave = self._selected_info.get_se_cosave_path()
-        return cosave.exists()
+        return self._selected_info.get_xse_cosave_path().exists()
 
     def Execute(self):
         with balt.BusyCursor():
             log = bolt.LogFile(StringIO.StringIO())
-            cosave = self._selected_info.get_cosave()
+            cosave = self._selected_info.get_xse_cosave()
             if cosave is not None:
-                cosave.logStatObse(log, self._selected_info.header.masters)
+                cosave.dump_to_log(log, self._selected_info.header.masters)
         text = log.out.getvalue()
         log.out.close()
-        self._showLog(text, title=self._selected_item.s, fixedFont=False)
+        if cosave is not None:
+            self._showLog(text, title=cosave.cosave_path.tail.s,
+                          fixedFont=False)
+
+#------------------------------------------------------------------------------
+class Save_StatPluggy(AppendableLink, OneItemLink):
+    """Dump Pluggy blocks from .pluggy files."""
+    _text = _(u'Dump .pluggy Contents')
+    _help = _(u'Dumps contents of associated Pluggy cosave into a log.')
+
+    def _append(self, window): return bush.game.has_standalone_pluggy
+
+    def _enable(self):
+        if not super(Save_StatPluggy, self)._enable(): return False
+        return self._selected_info.get_pluggy_cosave_path().exists()
+
+    def Execute(self):
+        with balt.BusyCursor():
+            log = bolt.LogFile(StringIO.StringIO())
+            cosave = self._selected_info.get_pluggy_cosave()
+            if cosave is not None:
+                cosave.dump_to_log(log, self._selected_info.header.masters)
+        text = log.out.getvalue()
+        log.out.close()
+        if cosave is not None:
+            self._showLog(text, title=cosave.cosave_path.tail.s,
+                          fixedFont=False)
 
 #------------------------------------------------------------------------------
 class Save_Unbloat(OneItemLink):

Mopy/bash/bolt.py

@@ -1649,6 +1649,7 @@ def unpack_str32(ins): return ins.read(struct_unpack('I', ins.read(4))[0])
 def unpack_int(ins): return struct_unpack('I', ins.read(4))[0]
 def unpack_short(ins): return struct_unpack('H', ins.read(2))[0]
 def unpack_float(ins): return struct_unpack('f', ins.read(4))[0]
+def unpack_double(ins): return struct_unpack('d', ins.read(8))[0]
 def unpack_byte(ins): return struct_unpack('B', ins.read(1))[0]
 def unpack_int_signed(ins): return struct_unpack('i', ins.read(4))[0]
 def unpack_int64_signed(ins): return struct_unpack('q', ins.read(8))[0]

Mopy/bash/bosh/__init__.py

@@ -273,7 +273,7 @@ def __init__(self, fullpath, load_cache=False):
 
     def _reset_masters(self):
         #--Master Names/Order
-        self.masterNames = tuple(self.header.masters)
+        self.masterNames = tuple(self.get_masters())
         self.masterOrder = tuple() #--Reset to empty for now
 
     def _file_changed(self, stat_tuple):
@@ -335,6 +335,18 @@ def getStatus(self):
         else:
             return status
 
+    def get_masters(self):
+        """
+        Returns the masters of this file as a list, if that operation makes any
+        sense. For example, the masters of a mod are its plugin masters, while
+        the masters of a save file are the plugins listed in its plugin list.
+        If this operation does not make sense (e.g. on an archive), an
+        AbstractError is raised.
+
+        :return: A list of the masters of this file, as paths.
+        """
+        raise AbstractError()
+
     # Backup stuff - beta, see #292 -------------------------------------------
     def getFileInfos(self):
         """Return one of the FileInfos singletons depending on fileInfo type.
@@ -508,6 +520,9 @@ def setmtime(self, set_time=0, crc_changed=False):
         else:
             self.calculate_crc(recalculate=True)
 
+    def get_masters(self):
+        return self.header.masters
+
     # Ghosting and ghosting related overrides ---------------------------------
     def do_update(self):
         self.isGhost, old_ghost = not self._abs_path.exists() and (
@@ -967,17 +982,14 @@ def listErrors(self):
 
 #------------------------------------------------------------------------------
 from .save_headers import get_save_header_type, SaveFileHeader
-from ._saves import PluggyFile
+from .cosaves import PluggyCosave
 from . import cosaves
 
 class SaveInfo(FileInfo):
-    _cosave_type = None  # type: cosaves.ACoSaveFile
+    # The xSE cosave that may come with this save file. Lazily initialized.
+    _xse_cosave = None
 
-    @property
-    def cosave_type(self):
-        if self._cosave_type is None:
-            SaveInfo._cosave_type = cosaves.get_cosave_type(bush.game.fsName)
-        return self._cosave_type
+    def cosave_type(self): return cosaves.get_cosave_type(bush.game.fsName)
 
     def getFileInfos(self): return saveInfos
 
@@ -1013,26 +1025,29 @@ def write_masters(self):
         oldMasters = [GPath(decode(x)) for x in oldMasters]
         self.abs_path.untemp()
         #--Cosaves
-        masterMap = dict(
-            (x, y) for x, y in zip(oldMasters, self.header.masters) if x != y)
-        #--Pluggy file?
-        pluggyPath = CoSaves.getPaths(self.abs_path)[0]
-        if masterMap and pluggyPath.exists():
-            pluggy = PluggyFile(pluggyPath)
-            pluggy.load()
-            pluggy.mapMasters(masterMap)
-            pluggy.safeSave()
-        #--OBSE/SKSE file?
-        cosave = self.get_cosave()
-        if cosave is not None:
-            cosave.map_masters(masterMap)
-            cosave.write_cosave_safe()
+        master_map = dict((x.s, y.s) for x, y in
+                          zip(oldMasters, self.header.masters) if x != y)
+        if master_map:
+            #--Pluggy cosave?
+            if bush.game.has_standalone_pluggy:
+                pluggy_path = self.get_pluggy_cosave_path()
+                if pluggy_path.isfile():
+                    pluggy_cosave = self.get_pluggy_cosave()
+                    pluggy_cosave.remap_plugins(master_map)
+                    pluggy_cosave.write_cosave_safe()
+            #--xSE cosave?
+            if bush.game.se.se_abbrev:
+                xse_path = self.get_xse_cosave_path()
+                if xse_path is not None and xse_path.isfile():
+                    xse_cosave = self.get_xse_cosave()
+                    xse_cosave.remap_plugins(master_map)
+                    xse_cosave.write_cosave_safe()
 
     def get_cosave_tags(self):
         """Return strings expressing whether cosaves exist and are correct."""
         cPluggy, cObse = (u'', u'')
         pluggy = self.name.root + u'.pluggy'
-        obse = self.get_se_cosave_path()
+        obse = self.get_xse_cosave_path()
         if pluggy.exists():
             cPluggy = u'XP'[abs(pluggy.mtime - self.mtime) < 10]
         if obse and obse.exists():
@@ -1044,21 +1059,52 @@ def backup_paths(self, first=False):
         save_paths.extend(CoSaves.get_new_paths(*save_paths[0]))
         return save_paths
 
-    def get_cosave(self):
-        """:rtype: cosaves.ACoSaveFile"""
-        cosave_path = self.get_se_cosave_path()
-        if cosave_path is None: return None
-        try:
-            return self.cosave_type(cosave_path) # type: cosaves.ACoSaveFile
-        except (OSError, IOError, FileError) as e:
-            if isinstance(e, FileError) or (
-                isinstance(e, (OSError, IOError)) and e.errno != errno.ENOENT):
-                deprint(u'Failed to open %s' % cosave_path, traceback=True)
-            return None
-
-    def get_se_cosave_path(self):
+    # TODO(inf) Turn into a property?
+    def get_xse_cosave(self):
+        """:rtype: cosaves.xSECosave"""
+        if self._xse_cosave is None:
+            cosave_path = self.get_xse_cosave_path()
+            if cosave_path is None: return None
+            try:
+                cosave_constructor = self.cosave_type()
+                self._xse_cosave = cosave_constructor(cosave_path)
+            except (OSError, IOError, FileError) as e:
+                if isinstance(e, FileError) or (
+                    isinstance(e, (OSError, IOError)) and e.errno != errno.ENOENT):
+                    deprint(u'Failed to open %s' % cosave_path, traceback=True)
+                return None
+        return self._xse_cosave
+
+    def get_xse_cosave_path(self):
         if self.cosave_type is None: return None
-        return self.getPath().root + u'.' + self.cosave_type.signature.lower()
+        return self.getPath().root + bush.game.se.cosave_ext
+
+    def get_pluggy_cosave(self):
+        cosave_path = self.get_pluggy_cosave_path()
+        if cosave_path is not None:
+            try:
+                return PluggyCosave(cosave_path)
+            except (OSError, IOError, FileError) as e:
+                if isinstance(e, FileError) or (
+                    isinstance(e, (OSError, IOError)) and e.errno != errno.ENOENT):
+                    deprint(u'Failed to open %s' % cosave_path, traceback=True)
+        return None
+
+    def get_pluggy_cosave_path(self):
+        return self.getPath().root + u'.pluggy'
+
+    def get_masters(self):
+        if bush.game.has_esl:
+            xse_cosave_path = self.get_xse_cosave_path()
+            if xse_cosave_path is not None and xse_cosave_path.isfile():
+                # Make sure the cosave's masters are actually useful
+                xse_cosave = self.get_xse_cosave()
+                if xse_cosave.has_accurate_master_list(True):
+                    return [GPath(master) for master in
+                            xse_cosave.get_master_list()]
+        # Fall back on the regular masters - either the cosave is unnecessary,
+        # doesn't exist or isn't accurate
+        return self.header.masters
 
 #------------------------------------------------------------------------------
 class DataStore(DataDict):

Mopy/bash/bosh/_saves.py

@@ -34,7 +34,7 @@
     unpack_short, struct_pack, struct_unpack
 from ..brec import ModReader, MreRecord, ModWriter, getObjectIndex, \
     getFormIndices
-from ..exception import FileError, ModError, StateError
+from ..exception import ModError, StateError
 from ..parsers import LoadFactory, ModFile
 
 #------------------------------------------------------------------------------
@@ -216,102 +216,6 @@ def dumpText(self,saveFile):
                         self.skills))
             return buff.getvalue()
 
-#------------------------------------------------------------------------------
-class PluggyFile:
-    """Represents a .pluggy cofile for saves. Used for editing masters list."""
-    def __init__(self,path):
-        self.path = path
-        self.name = path.tail
-        self.tag = None
-        self.version = None
-        self._plugins = None
-        self.other = None
-        self.valid = False
-
-    def mapMasters(self,masterMap):
-        """Update plugin names according to masterMap."""
-        if not self.valid:
-            raise FileError(self.path.tail, u"File not initialized.")
-        self._plugins = [(x, y, masterMap.get(z,z)) for x,y,z in self._plugins]
-
-    def load(self):
-        """Read file."""
-        import binascii
-        path_size = self.path.size
-        with self.path.open('rb') as ins:
-            buff = ins.read(path_size-4)
-            crc32, = struct_unpack('=i', ins.read(4))
-        crcNew = binascii.crc32(buff)
-        if crc32 != crcNew:
-            raise FileError(self.path.tail,
-                            u'CRC32 file check failed. File: %X, Calc: %X' % (
-                                crc32, crcNew))
-        #--Header
-        with sio(buff) as ins:
-            def _unpack(fmt, fmt_siz):
-                return struct_unpack(fmt, ins.read(fmt_siz))
-            if ins.read(10) != 'PluggySave':
-                raise FileError(self.path.tail, u'File tag != "PluggySave"')
-            self.version, = _unpack('I',4)
-            #--Reject versions earlier than 1.02
-            if self.version < 0x01020000:
-                raise FileError(self.path.tail,
-                                u'Unsupported file version: %X' % self.version)
-            #--Plugins
-            self._plugins = []
-            type, = _unpack('=B',1)
-            if type != 0:
-                raise FileError(self.path.tail,
-                                u'Expected plugins record, but got %d.' % type)
-            count, = _unpack('=I',4)
-            for x in range(count):
-                espid,index,modLen = _unpack('=2BI',6)
-                modName = GPath(decode(ins.read(modLen)))
-                self._plugins.append((espid, index, modName))
-            #--Other
-            self.other = ins.getvalue()[ins.tell():]
-        deprint(struct_unpack('I', self.other[-4:]), self.path.size-8)
-        #--Done
-        self.valid = True
-
-    def save(self,path=None,mtime=0):
-        """Saves."""
-        import binascii
-        if not self.valid:
-            raise FileError(self.path.tail, u"File not initialized.")
-        #--Buffer
-        with sio() as buff:
-            #--Save
-            def _pack(fmt, *args):
-                buff.write(struct_pack(fmt, *args))
-            buff.write('PluggySave')
-            _pack('=I',self.version)
-            #--Plugins
-            _pack('=B',0)
-            _pack('=I', len(self._plugins))
-            for (espid,index,modName) in self._plugins:
-                modName = encode(modName.cs)
-                _pack('=2BI',espid,index,len(modName))
-                buff.write(modName)
-            #--Other
-            buff.write(self.other)
-            #--End control
-            buff.seek(-4,1)
-            _pack('=I',buff.tell())
-            #--Save
-            path = path or self.path
-            mtime = mtime or path.exists() and path.mtime
-            text = buff.getvalue()
-            with path.open('wb') as out:
-                out.write(text)
-                out.write(struct_pack('i', binascii.crc32(text)))
-        path.mtime = mtime
-
-    def safeSave(self):
-        """Save data to file safely."""
-        self.save(self.path.temp,self.path.mtime)
-        self.path.untemp()
-
 # Save File -------------------------------------------------------------------
 class SaveFile:
     """Represents a Tes4 Save file."""