kindle manager

mobimaster/mobi/mobi_split.py

@@ -0,0 +1,505 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+# vim:ts=4:sw=4:softtabstop=4:smarttab:expandtab
+
+from __future__ import unicode_literals, division, absolute_import, print_function
+from loguru import logger
+
+import struct
+
+# note:  struct pack, unpack, unpack_from all require bytestring format
+# data all the way up to at least python 2.7.5, python 3 okay with bytestring
+
+from .unipath import pathof
+
+
+# CG : reference https://wiki.mobileread.com/wiki/MOBI
+# important  pdb header offsets
+unique_id_seed = 68
+number_of_pdb_records = 76
+
+# important palmdoc header offsets
+book_length = 4
+book_record_count = 8
+first_pdb_record = 78
+
+# important rec0 offsets
+length_of_book = 4
+mobi_header_base = 16
+mobi_header_length = 20
+mobi_type = 24
+mobi_version = 36
+first_non_text = 80
+title_offset = 84
+first_resc_record = 108
+first_content_index = 192
+last_content_index = 194
+kf8_fdst_index = 192  # for KF8 mobi headers
+fcis_index = 200
+flis_index = 208
+srcs_index = 224
+srcs_count = 228
+primary_index = 244
+datp_index = 256
+huffoff = 112
+hufftbloff = 120
+
+
+def getint(datain, ofs, sz=b"L"):
+    (i,) = struct.unpack_from(b">" + sz, datain, ofs)
+    return i
+
+
+def writeint(datain, ofs, n, len=b"L"):
+    if len == b"L":
+        return datain[:ofs] + struct.pack(b">L", n) + datain[ofs + 4 :]
+    else:
+        return datain[:ofs] + struct.pack(b">H", n) + datain[ofs + 2 :]
+
+
+def getsecaddr(datain, secno):
+    nsec = getint(datain, number_of_pdb_records, b"H")
+    assert secno >= 0 & secno < nsec, "secno %d out of range (nsec=%d)" % (secno, nsec)
+    secstart = getint(datain, first_pdb_record + secno * 8)
+    if secno == nsec - 1:
+        secend = len(datain)
+    else:
+        secend = getint(datain, first_pdb_record + (secno + 1) * 8)
+    return secstart, secend
+
+
+def readsection(datain, secno):
+    secstart, secend = getsecaddr(datain, secno)
+    return datain[secstart:secend]
+
+
+def writesection(datain, secno, secdata):  # overwrite, accounting for different length
+    # dataout = deletesectionrange(datain,secno, secno)
+    # return insertsection(dataout, secno, secdata)
+    datalst = []
+    nsec = getint(datain, number_of_pdb_records, b"H")
+    zerosecstart, zerosecend = getsecaddr(datain, 0)
+    secstart, secend = getsecaddr(datain, secno)
+    dif = len(secdata) - (secend - secstart)
+    datalst.append(datain[:unique_id_seed])
+    datalst.append(struct.pack(b">L", 2 * nsec + 1))
+    datalst.append(datain[unique_id_seed + 4 : number_of_pdb_records])
+    datalst.append(struct.pack(b">H", nsec))
+    newstart = zerosecstart
+    for i in range(0, secno):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    datalst.append(struct.pack(b">L", secstart) + struct.pack(b">L", (2 * secno)))
+    for i in range(secno + 1, nsec):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs = ofs + dif
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    lpad = newstart - (first_pdb_record + 8 * nsec)
+    if lpad > 0:
+        datalst.append(b"\0" * lpad)
+    datalst.append(datain[zerosecstart:secstart])
+    datalst.append(secdata)
+    datalst.append(datain[secend:])
+    dataout = b"".join(datalst)
+    return dataout
+
+
+def nullsection(datain, secno):  # make it zero-length without deleting it
+    datalst = []
+    nsec = getint(datain, number_of_pdb_records, b"H")
+    secstart, secend = getsecaddr(datain, secno)
+    zerosecstart, zerosecend = getsecaddr(datain, 0)
+    dif = secend - secstart
+    datalst.append(datain[:first_pdb_record])
+    for i in range(0, secno + 1):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    for i in range(secno + 1, nsec):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs = ofs - dif
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    lpad = zerosecstart - (first_pdb_record + 8 * nsec)
+    if lpad > 0:
+        datalst.append(b"\0" * lpad)
+    datalst.append(datain[zerosecstart:secstart])
+    datalst.append(datain[secend:])
+    dataout = b"".join(datalst)
+    return dataout
+
+
+def deletesectionrange(datain, firstsec, lastsec):  # delete a range of sections
+    datalst = []
+    firstsecstart, firstsecend = getsecaddr(datain, firstsec)
+    lastsecstart, lastsecend = getsecaddr(datain, lastsec)
+    zerosecstart, zerosecend = getsecaddr(datain, 0)
+    dif = lastsecend - firstsecstart + 8 * (lastsec - firstsec + 1)
+    nsec = getint(datain, number_of_pdb_records, b"H")
+    datalst.append(datain[:unique_id_seed])
+    datalst.append(struct.pack(b">L", 2 * (nsec - (lastsec - firstsec + 1)) + 1))
+    datalst.append(datain[unique_id_seed + 4 : number_of_pdb_records])
+    datalst.append(struct.pack(b">H", nsec - (lastsec - firstsec + 1)))
+    newstart = zerosecstart - 8 * (lastsec - firstsec + 1)
+    for i in range(0, firstsec):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs = ofs - 8 * (lastsec - firstsec + 1)
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    for i in range(lastsec + 1, nsec):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs = ofs - dif
+        flgval = 2 * (i - (lastsec - firstsec + 1))
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    lpad = newstart - (first_pdb_record + 8 * (nsec - (lastsec - firstsec + 1)))
+    if lpad > 0:
+        datalst.append(b"\0" * lpad)
+    datalst.append(datain[zerosecstart:firstsecstart])
+    datalst.append(datain[lastsecend:])
+    dataout = b"".join(datalst)
+    return dataout
+
+
+def insertsection(datain, secno, secdata):  # insert a new section
+    datalst = []
+    nsec = getint(datain, number_of_pdb_records, b"H")
+    # print("inserting secno" , secno,  "into" ,nsec, "sections")
+    secstart, secend = getsecaddr(datain, secno)
+    zerosecstart, zerosecend = getsecaddr(datain, 0)
+    dif = len(secdata)
+    datalst.append(datain[:unique_id_seed])
+    datalst.append(struct.pack(b">L", 2 * (nsec + 1) + 1))
+    datalst.append(datain[unique_id_seed + 4 : number_of_pdb_records])
+    datalst.append(struct.pack(b">H", nsec + 1))
+    newstart = zerosecstart + 8
+    for i in range(0, secno):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs += 8
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    datalst.append(struct.pack(b">L", secstart + 8) + struct.pack(b">L", (2 * secno)))
+    for i in range(secno, nsec):
+        ofs, flgval = struct.unpack_from(b">2L", datain, first_pdb_record + i * 8)
+        ofs = ofs + dif + 8
+        flgval = 2 * (i + 1)
+        datalst.append(struct.pack(b">L", ofs) + struct.pack(b">L", flgval))
+    lpad = newstart - (first_pdb_record + 8 * (nsec + 1))
+    if lpad > 0:
+        datalst.append(b"\0" * lpad)
+    datalst.append(datain[zerosecstart:secstart])
+    datalst.append(secdata)
+    datalst.append(datain[secstart:])
+    dataout = b"".join(datalst)
+    return dataout
+
+
+def insertsectionrange(
+    sectionsource, firstsec, lastsec, sectiontarget, targetsec
+):  # insert a range of sections
+    # print("inserting secno" , firstsec,  "to", lastsec, "into" ,targetsec, "sections")
+    # dataout = sectiontarget
+    # for idx in range(lastsec,firstsec-1,-1):
+    #    dataout = insertsection(dataout,targetsec,readsection(sectionsource,idx))
+    # return dataout
+    datalst = []
+    nsec = getint(sectiontarget, number_of_pdb_records, b"H")
+    zerosecstart, zerosecend = getsecaddr(sectiontarget, 0)
+    insstart, nul = getsecaddr(sectiontarget, targetsec)
+    nins = lastsec - firstsec + 1
+    srcstart, nul = getsecaddr(sectionsource, firstsec)
+    nul, srcend = getsecaddr(sectionsource, lastsec)
+    newstart = zerosecstart + 8 * nins
+
+    datalst.append(sectiontarget[:unique_id_seed])
+    datalst.append(struct.pack(b">L", 2 * (nsec + nins) + 1))
+    datalst.append(sectiontarget[unique_id_seed + 4 : number_of_pdb_records])
+    datalst.append(struct.pack(b">H", nsec + nins))
+    for i in range(0, targetsec):
+        ofs, flgval = struct.unpack_from(
+            b">2L", sectiontarget, first_pdb_record + i * 8
+        )
+        ofsnew = ofs + 8 * nins
+        flgvalnew = flgval
+        datalst.append(struct.pack(b">L", ofsnew) + struct.pack(b">L", flgvalnew))
+        # print(ofsnew, flgvalnew, ofs, flgval)
+    srcstart0, nul = getsecaddr(sectionsource, firstsec)
+    for i in range(nins):
+        isrcstart, nul = getsecaddr(sectionsource, firstsec + i)
+        ofsnew = insstart + (isrcstart - srcstart0) + 8 * nins
+        flgvalnew = 2 * (targetsec + i)
+        datalst.append(struct.pack(b">L", ofsnew) + struct.pack(b">L", flgvalnew))
+        # print(ofsnew, flgvalnew)
+    dif = srcend - srcstart
+    for i in range(targetsec, nsec):
+        ofs, flgval = struct.unpack_from(
+            b">2L", sectiontarget, first_pdb_record + i * 8
+        )
+        ofsnew = ofs + dif + 8 * nins
+        flgvalnew = 2 * (i + nins)
+        datalst.append(struct.pack(b">L", ofsnew) + struct.pack(b">L", flgvalnew))
+        # print(ofsnew, flgvalnew, ofs, flgval)
+    lpad = newstart - (first_pdb_record + 8 * (nsec + nins))
+    if lpad > 0:
+        datalst.append(b"\0" * lpad)
+    datalst.append(sectiontarget[zerosecstart:insstart])
+    datalst.append(sectionsource[srcstart:srcend])
+    datalst.append(sectiontarget[insstart:])
+    dataout = b"".join(datalst)
+    return dataout
+
+
+def get_exth_params(rec0):
+    ebase = mobi_header_base + getint(rec0, mobi_header_length)
+    elen = getint(rec0, ebase + 4)
+    enum = getint(rec0, ebase + 8)
+    return ebase, elen, enum
+
+
+def add_exth(rec0, exth_num, exth_bytes):
+    ebase, elen, enum = get_exth_params(rec0)
+    newrecsize = 8 + len(exth_bytes)
+    newrec0 = (
+        rec0[0 : ebase + 4]
+        + struct.pack(b">L", elen + newrecsize)
+        + struct.pack(b">L", enum + 1)
+        + struct.pack(b">L", exth_num)
+        + struct.pack(b">L", newrecsize)
+        + exth_bytes
+        + rec0[ebase + 12 :]
+    )
+    newrec0 = writeint(
+        newrec0, title_offset, getint(newrec0, title_offset) + newrecsize
+    )
+    return newrec0
+
+
+def read_exth(rec0, exth_num):
+    exth_values = []
+    ebase, elen, enum = get_exth_params(rec0)
+    ebase = ebase + 12
+    while enum > 0:
+        exth_id = getint(rec0, ebase)
+        if exth_id == exth_num:
+            # We might have multiple exths, so build a list.
+            exth_values.append(rec0[ebase + 8 : ebase + getint(rec0, ebase + 4)])
+        enum = enum - 1
+        ebase = ebase + getint(rec0, ebase + 4)
+    return exth_values
+
+
+def write_exth(rec0, exth_num, exth_bytes):
+    ebase, elen, enum = get_exth_params(rec0)
+    ebase_idx = ebase + 12
+    enum_idx = enum
+    while enum_idx > 0:
+        exth_id = getint(rec0, ebase_idx)
+        if exth_id == exth_num:
+            dif = len(exth_bytes) + 8 - getint(rec0, ebase_idx + 4)
+            newrec0 = rec0
+            if dif != 0:
+                newrec0 = writeint(
+                    newrec0, title_offset, getint(newrec0, title_offset) + dif
+                )
+            return (
+                newrec0[: ebase + 4]
+                + struct.pack(
+                    b">L", elen + len(exth_bytes) + 8 - getint(rec0, ebase_idx + 4)
+                )
+                + struct.pack(b">L", enum)
+                + rec0[ebase + 12 : ebase_idx + 4]
+                + struct.pack(b">L", len(exth_bytes) + 8)
+                + exth_bytes
+                + rec0[ebase_idx + getint(rec0, ebase_idx + 4) :]
+            )
+        enum_idx = enum_idx - 1
+        ebase_idx = ebase_idx + getint(rec0, ebase_idx + 4)
+    return rec0
+
+
+def del_exth(rec0, exth_num):
+    ebase, elen, enum = get_exth_params(rec0)
+    ebase_idx = ebase + 12
+    enum_idx = 0
+    while enum_idx < enum:
+        exth_id = getint(rec0, ebase_idx)
+        exth_size = getint(rec0, ebase_idx + 4)
+        if exth_id == exth_num:
+            newrec0 = rec0
+            newrec0 = writeint(
+                newrec0, title_offset, getint(newrec0, title_offset) - exth_size
+            )
+            newrec0 = newrec0[:ebase_idx] + newrec0[ebase_idx + exth_size :]
+            newrec0 = (
+                newrec0[0 : ebase + 4]
+                + struct.pack(b">L", elen - exth_size)
+                + struct.pack(b">L", enum - 1)
+                + newrec0[ebase + 12 :]
+            )
+            return newrec0
+        enum_idx += 1
+        ebase_idx = ebase_idx + exth_size
+    return rec0
+
+
+class mobi_split:
+    def __init__(self, infile):
+        datain = b""
+        with open(pathof(infile), "rb") as f:
+            datain = f.read()
+        datain_rec0 = readsection(datain, 0)
+        ver = getint(datain_rec0, mobi_version)
+        self.combo = ver != 8
+        if not self.combo:
+            return
+        exth121 = read_exth(datain_rec0, 121)
+        if len(exth121) == 0:
+            self.combo = False
+            return
+        else:
+            # only pay attention to first exth121
+            # (there should only be one)
+            (datain_kf8,) = struct.unpack_from(b">L", exth121[0], 0)
+            if datain_kf8 == 0xFFFFFFFF:
+                self.combo = False
+                return
+        datain_kfrec0 = readsection(datain, datain_kf8)
+
+        # create the standalone mobi7
+        num_sec = getint(datain, number_of_pdb_records, b"H")
+        # remove BOUNDARY up to but not including ELF record
+        self.result_file7 = deletesectionrange(datain, datain_kf8 - 1, num_sec - 2)
+        # check if there are SRCS records and delete them
+        srcs = getint(datain_rec0, srcs_index)
+        num_srcs = getint(datain_rec0, srcs_count)
+        if srcs != 0xFFFFFFFF and num_srcs > 0:
+            self.result_file7 = deletesectionrange(
+                self.result_file7, srcs, srcs + num_srcs - 1
+            )
+            datain_rec0 = writeint(datain_rec0, srcs_index, 0xFFFFFFFF)
+            datain_rec0 = writeint(datain_rec0, srcs_count, 0)
+        # reset the EXTH 121 KF8 Boundary meta data to 0xffffffff
+        datain_rec0 = write_exth(datain_rec0, 121, struct.pack(b">L", 0xFFFFFFFF))
+        # datain_rec0 = del_exth(datain_rec0,121)
+        # datain_rec0 = del_exth(datain_rec0,534)
+        # don't remove the EXTH 125 KF8 Count of Resources, seems to be present in mobi6 files as well
+        # set the EXTH 129 KF8 Masthead / Cover Image string to the null string
+        datain_rec0 = write_exth(datain_rec0, 129, b"")
+        # don't remove the EXTH 131 KF8 Unidentified Count, seems to be present in mobi6 files as well
+
+        # need to reset flags stored in 0x80-0x83
+        # old mobi with exth: 0x50, mobi7 part with exth: 0x1850, mobi8 part with exth: 0x1050
+        # Bit Flags
+        # 0x1000 = Bit 12 indicates if embedded fonts are used or not
+        # 0x0800 = means this Header points to *shared* images/resource/fonts ??
+        # 0x0080 = unknown new flag, why is this now being set by Kindlegen 2.8?
+        # 0x0040 = exth exists
+        # 0x0010 = Not sure but this is always set so far
+        (fval,) = struct.unpack_from(b">L", datain_rec0, 0x80)
+        # need to remove flag 0x0800 for KindlePreviewer 2.8 and unset Bit 12 for embedded fonts
+        fval = fval & 0x07FF
+        datain_rec0 = datain_rec0[:0x80] + struct.pack(b">L", fval) + datain_rec0[0x84:]
+
+        self.result_file7 = writesection(self.result_file7, 0, datain_rec0)
+
+        # no need to replace kf8 style fcis with mobi 7 one
+        # fcis_secnum, = struct.unpack_from(b'>L',datain_rec0, 0xc8)
+        # if fcis_secnum != 0xffffffff:
+        #     fcis_info = readsection(datain, fcis_secnum)
+        #     text_len,  = struct.unpack_from(b'>L', fcis_info, 0x14)
+        #     new_fcis = 'FCIS\x00\x00\x00\x14\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00'
+        #     new_fcis += struct.pack(b'>L',text_len)
+        #     new_fcis += '\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x08\x00\x01\x00\x01\x00\x00\x00\x00'
+        #     self.result_file7 = writesection(self.result_file7, fcis_secnum, new_fcis)
+
+        firstimage = getint(datain_rec0, first_resc_record)
+        lastimage = getint(datain_rec0, last_content_index, b"H")
+        # print("Old First Image, last Image", firstimage,lastimage)
+        if lastimage == 0xFFFF:
+            # find the lowest of the next sections and copy up to that.
+            ofs_list = [
+                (fcis_index, b"L"),
+                (flis_index, b"L"),
+                (datp_index, b"L"),
+                (hufftbloff, b"L"),
+            ]
+            for ofs, sz in ofs_list:
+                n = getint(datain_rec0, ofs, sz)
+                # print("n",n)
+                if n > 0 and n < lastimage:
+                    lastimage = n - 1
+        logger.debug("First Image, last Image %s %s" % (firstimage, lastimage))
+
+        # Try to null out FONT and RES, but leave the (empty) PDB record so image refs remain valid
+        for i in range(firstimage, lastimage):
+            imgsec = readsection(self.result_file7, i)
+            if imgsec[0:4] in [b"RESC", b"FONT"]:
+                self.result_file7 = nullsection(self.result_file7, i)
+
+        # mobi7 finished
+
+        # create standalone mobi8
+        self.result_file8 = deletesectionrange(datain, 0, datain_kf8 - 1)
+        target = getint(datain_kfrec0, first_resc_record)
+        self.result_file8 = insertsectionrange(
+            datain, firstimage, lastimage, self.result_file8, target
+        )
+        datain_kfrec0 = readsection(self.result_file8, 0)
+
+        # Only keep the correct EXTH 116 StartOffset, KG 2.5 carries over the one from the mobi7 part, which then points at garbage in the mobi8 part, and confuses FW 3.4
+        kf8starts = read_exth(datain_kfrec0, 116)
+        # If we have multiple StartOffset, keep only the last one
+        kf8start_count = len(kf8starts)
+        while kf8start_count > 1:
+            kf8start_count -= 1
+            datain_kfrec0 = del_exth(datain_kfrec0, 116)
+
+        # update the EXTH 125 KF8 Count of Images/Fonts/Resources
+        datain_kfrec0 = write_exth(
+            datain_kfrec0, 125, struct.pack(b">L", lastimage - firstimage + 1)
+        )
+
+        # need to reset flags stored in 0x80-0x83
+        # old mobi with exth: 0x50, mobi7 part with exth: 0x1850, mobi8 part with exth: 0x1050
+        # standalone mobi8 with exth: 0x0050
+        # Bit Flags
+        # 0x1000 = Bit 12 indicates if embedded fonts are used or not
+        # 0x0800 = means this Header points to *shared* images/resource/fonts ??
+        # 0x0080 = unknown new flag, why is this now being set by Kindlegen 2.8?
+        # 0x0040 = exth exists
+        # 0x0010 = Not sure but this is always set so far
+        (fval,) = struct.unpack_from(">L", datain_kfrec0, 0x80)
+        fval = fval & 0x1FFF
+        fval |= 0x0800
+        datain_kfrec0 = (
+            datain_kfrec0[:0x80] + struct.pack(b">L", fval) + datain_kfrec0[0x84:]
+        )
+
+        # properly update other index pointers that have been shifted by the insertion of images
+        ofs_list = [
+            (kf8_fdst_index, b"L"),
+            (fcis_index, b"L"),
+            (flis_index, b"L"),
+            (datp_index, b"L"),
+            (hufftbloff, b"L"),
+        ]
+        for ofs, sz in ofs_list:
+            n = getint(datain_kfrec0, ofs, sz)
+            if n != 0xFFFFFFFF:
+                datain_kfrec0 = writeint(
+                    datain_kfrec0, ofs, n + lastimage - firstimage + 1, sz
+                )
+        self.result_file8 = writesection(self.result_file8, 0, datain_kfrec0)
+
+        # no need to replace kf8 style fcis with mobi 7 one
+        # fcis_secnum, = struct.unpack_from(b'>L',datain_kfrec0, 0xc8)
+        # if fcis_secnum != 0xffffffff:
+        #     fcis_info = readsection(self.result_file8, fcis_secnum)
+        #     text_len,  = struct.unpack_from(b'>L', fcis_info, 0x14)
+        #     new_fcis = 'FCIS\x00\x00\x00\x14\x00\x00\x00\x10\x00\x00\x00\x01\x00\x00\x00\x00'
+        #     new_fcis += struct.pack(b'>L',text_len)
+        #     new_fcis += '\x00\x00\x00\x00\x00\x00\x00\x20\x00\x00\x00\x08\x00\x01\x00\x01\x00\x00\x00\x00'
+        #     self.result_file8 = writesection(self.result_file8, fcis_secnum, new_fcis)
+
+        # mobi8 finished
+
+    def getResult8(self):
+        return self.result_file8
+
+    def getResult7(self):
+        return self.result_file7