kindle manager

changelog.md

@@ -92,6 +92,7 @@ b['1']['2'] = {'3':1}  # OK
     - kindle **done**
     - duokan
     - amazon
+- link to douban
 - export function:
     - to [evernote](https://github.com/benhorvath/kindle2evernote/blob/master/kindle2evernote.py)
     - to web html format
@@ -102,11 +103,14 @@ b['1']['2'] = {'3':1}  # OK
 - easily coly filter notes to clipboard
 - implement command line based on [argparse](https://www.jianshu.com/p/a41fbd4919f8)
 - mobi / epub reader **XXXX**
-- convert epub to mobi
+- import .azw3/.epub/.mobi file to kindle
+- [convert](https://github.com/jianyun8023/convert-ebook) .epub -> .mobi <br>
+          .azw3 -> .mobi <br>
+          .md   -> .mobi
 - parse /Volumes/Kindle/system/vocabulary/vocab.db
 - [hotkey](https://toolinbox.net/Klib/)
 - write [tutorial](https://help.knotesapp.com/docs/tutorial/)
 - search word in online dictionary
 - chrome extension
-
+- clean (sdr)
-
+- sync modify to kindle device

kman.py

@@ -19,7 +19,7 @@ from collections import defaultdict
 
 # data structure - use dict
 '''
-books =
+books_data =
 {
     "bookname_xxx": {
         "author": "李",
@@ -56,16 +56,6 @@ books =
 }
 '''
 
-'''
-# vocab.db -> table: book_info
-bookname = {'id':{
-       'title':'bookname_xxx',
-       'authors':'xxx'
-       },}
-words = {
-    "bookname_xxx": {
-'''
-
 # modi clippath for different os
 SYS = 'WIN' if platform.system()=='Windows' else \
    ('LINUX' if platform.system()=='LINUX' else 'MAC')
@@ -73,14 +63,16 @@ SYS = 'WIN' if platform.system()=='Windows' else \
 # some constants
 LASTLINE = '=========='
 NTPREF   = '--CG注:'
-KINDLEFN = 'My Clippings.txt'
+CLIPFN   = 'My Clippings.txt'
+WORDFN   = 'vocab.db'
 CLIPPATH = './'
 OUTPREF  = './clip'
 DEBUG    = 1   # 0 - INFO; 1 - DEBUG
 LOG2FILE = 1   # 0 - to stdio; 1 - to file
 LOGFILE  = 'log'
 DELIMITER= '|'
-BACKUPFN = './backup/bk.data'
+BACKUPNOTEFN = './backup/bk.note.data'
+BACKUPWORDFN = './backup/bk.word.data'
 #HEADER   = {0:'type',1:'bookname',2:'author',3:'position',4:'date',5:'content'}
 
 # log info
@@ -137,7 +129,7 @@ class kMan:
         kp = self.get_kindle_path()
 
         if not kp:
-            s2 = u'Disconnected ({})'.format(CLIPPATH+KINDLEFN)
+            s2 = u'Disconnected ({})'.format(CLIPPATH+CLIPFN)
         else:
             with open(kp+'/system/version.txt' , 'r', encoding='utf8', errors='ignore') as f:
                 s2 = u'Connected ({}) version {}'.format(kp,f.read().strip())
@@ -385,7 +377,8 @@ class kMan:
                 0: root item clicked
                 1: bookname item clicked
                 2: author item clicked
-        Return: list:
+        Return: list: [filter books, filter seclist]
+            filter_seclist:
             [[Type,Bookname,Author,Position,Date,content],
              [Type,Bookname,Author,Position,Date,content]
             ....]
@@ -536,14 +529,14 @@ class kMan:
 
         return False
 
-    def import_clips(self, fp=(CLIPPATH+KINDLEFN)):
+    def import_clips(self, fp=(CLIPPATH+CLIPFN)):
         """import clips from local file or kindle
         4 lines for each section seperated with '======='
         so read 4 lines before '======='
 
         Args: fp - file path
         Return: 0 - want to import kindle but kindle is not connected
-                books dict
+                books_data dict
         """
         # check kindle by user just call get_kindle_path()
         """
@@ -608,6 +601,120 @@ class kMan:
 
         return self.drop_duplicate(bks)
 
+    # words operations
+    def import_words(self, fp=(CLIPPATH+WORDFN)):
+        """import words from local file or kindle
+           vocab.db -> tables: BOOK_INFO, LOOKUPS, WORDS
+
+        Args: fp - file path
+        Return: words_data = [bookinfo, words, lookups]
+                bookinfo = { 'book_key': {
+                            'bookname':'bookname_xxx',
+                            'author':'author_xxx'},}
+                   words = { 'word_id': {
+                            'word':'梅花鹿',
+                            'category':0},
+                            }
+                 lookups = [['id','word_key','book_key','pos','usage','timestamp'],
+                            ['id','word_key','book_key','pos','usage','timestamp'],]
+        """
+        import sqlite3
+        conn = sqlite3.connect(fp)
+
+        #[bookinfo, words, lookups] = [defaultdict(dict),defaultdict(dict),defaultdict(dict)]
+        [bookinfo, words, lookups] = [defaultdict(dict),defaultdict(dict),[]]
+
+        # select records from vocab.db -> tables: BOOK_INFO, LOOKUPS, WORDS
+        try:
+            cursor = conn.cursor()
+            cursor.execute('''select * from BOOK_INFO;''')
+            for row in cursor.fetchall():
+                bookinfo[row[0]]['bookname'] = row[4].replace(' ','')
+                bookinfo[row[0]]['author'] = row[5]
+                bookinfo[row[0]]['lang'] = row[5]
+
+            cursor.execute('''select * from WORDS;''')
+            for row in cursor.fetchall():
+                words[row[0]]['word'] = row[1]
+                #words[row[0]]['lang'] = row[3]
+                words[row[0]]['category'] = row[4]
+
+            cursor.execute('''select * from LOOKUPS;''')
+            for row in cursor.fetchall():
+                timearr = time.localtime(row[6]/1000.) # NOTE: length of timestring is 13
+                mtime = time.strftime("%Y/%m/%d %H:%M:%S", timearr)
+                """
+                lookups[row[0]]['word_key'] = row[1]
+                lookups[row[0]]['book_key'] = row[2]
+                lookups[row[0]]['pos'] = row[4]
+                lookups[row[0]]['usage'] = row[5]
+                lookups[row[0]]['timestamp'] = mtime 
+                """
+                lookups.append([row[0],row[1],row[2],row[4],row[5],mtime])
+
+        except Exception as e:
+            print('sql failure {}'.format(e))
+        finally:
+            conn.close()
+
+        words_data = [bookinfo, words, lookups]
+
+        return words_data
+
+    def filter_words(self, wdata, info=None, tp=0):
+        """ filter words to show the clips in table view
+        Args:
+            wdata = [bookinfo, words, lookups]
+            info: filter by bookname or word information
+            tp: type to be filter
+                0: bookname item clicked
+                1: word item clicked
+        Return: [filtted words_data, word_list]
+            word_list:
+            [[Bookname,Author,word,category],
+             [Bookname,Author,word,status],
+            ....]
+        """
+        [bookinfo, words, lookups] = wdata
+        word_list = []
+        # bookname tree clicked
+        if tp == 0:
+            for row in lookups:
+                word     = words[row[1]]['word']
+                category = words[row[1]]['category']
+                bookname = bookinfo[row[2]]['bookname']
+                author   = bookinfo[row[2]]['author']
+                if bookname==info or info == None:
+                    # maybe have duplication records
+                    word_list.append((bookname,author,word,category))
+        # word tree clicked
+        elif tp == 1:
+            pass
+        else:
+            return
+
+        # remove duplication
+        return list(set(word_list))
+
+    def get_lookups_word_num(self, wdata):
+        pass
+
+    def get_word_num(self, wdata):
+        pass
+
+    # show in tree view
+    def get_book_word_num(self,wdata):
+        [bookinfo, words, lookups] = wdata
+        bk_wd_num = defaultdict(dict)
+        nu = 0
+        for row in lookups:
+            nu += 1
+            word     = words[row[1]]['word']
+            bookname = bookinfo[row[2]]['bookname']
+            bk_wd_num.setdefault(bookname,0)
+            bk_wd_num[bookname] += 1
+
+        return [nu, bk_wd_num]
 
 if __name__=='__main__':
     #books = defaultdict(dict)

kmanapp.py

@@ -27,6 +27,20 @@ from kman import *
 # import binary resource file(kmanapp_rc.py)
 import kmanapp_rc
 
+notes_temp = """<br><span style='font-size:22pt;color:maroon'>《{bookname}》</span>
+               <span style='font-size:22pt;color:maroon'> {author} </span>
+               <span style='font-size:22pt;color:maroon'> ({time}) </span>
+               <span style='font-size:11pt;color:maroon'> 【{note}】 </span><hr><br><br>
+               <span style='font-size:22pt;color:#31849B'>{content}</span>
+               <span style='font-size:11pt;color:maroon'>【P{position}】</span><br>"""
+
+words_temp = """<br><span style='font-size:22pt;color:maroon'>{usage}</span><br><br>
+               <span style='font-size:12pt;color:maroon'> {bookname} </span>
+               <span style='font-size:12pt;color:maroon'> {author} </span>
+               <span style='font-size:12pt;color:maroon'> {category} </span>
+               <span style='font-size:12pt;color:maroon'> {timestamp} </span>
+               <span style='font-size:12pt;color:#31849B'>{position}</span><hr><br><br>"""
+
 ONLY_TEST = 1
 
 class kmanWindow(QMainWindow):
@@ -45,19 +59,34 @@ class kmanWindow(QMainWindow):
 
         self.add_ui_component()
 
+        # initial tree selected
+        self.tree_selected = 'note_root'
+
         self.km = kMan()
         # initial check order:
         #     1. backup file bk.data ->
         #     2. kindle(My Clippings.txt) ->
         #     3. local file(config) ->
-        if os.path.exists(BACKUPFN) and os.path.getsize(BACKUPFN) != 0:
+        flg = 0
-            self.books = self.km.json2dict(BACKUPFN)
+        if  os.path.exists(BACKUPNOTEFN) and os.path.exists(BACKUPWORDFN):
-        elif self.km.get_kindle_path():
+            self.books_data = self.km.json2dict(BACKUPNOTEFN)
+            self.words_data = self.km.json2dict(BACKUPWORDFN)
+            if (len(self.books_data)*len(self.words_data[0]))>=1:
+                self.books_data = self.km.json2dict(BACKUPNOTEFN)
+                self.words_data = self.km.json2dict(BACKUPWORDFN)
+                flg = 1
+
+        if self.km.get_kindle_path() and (not flg):
             self.import_kindle()
         else:
-            self.books = self.km.import_clips()
+            self.books_data = self.km.import_clips()
-        [self.filter_books, self.filter_data] = self.km.filter_clips(self.books)
+            self.words_data = self.km.import_words()
-        self.refresh_ui_component()
+
+        [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data)
+        #self.filter_list = self.km.filter_words(self.words_data)
+
+        self.fill_treeview()
+        self.refresh_ui_component(comp=1)
 
         # timer to check status of kindle
         self.timer = QTimer(self)
@@ -86,9 +115,11 @@ class kmanWindow(QMainWindow):
         ui.tableView.setColumnWidth(2, 50) # author
         ui.tableView.selectRow(0)
 
-        # creat tablemodel
+        # initial tableView
-        data = self.convert_to_panda(self.filter_data)
+        data = self.convert_to_panda(self.filter_list)
         ui.tablemodel = nTableModel(data)
+        ui.tableView.verticalHeader().hide()
+        ui.tableView.setModel(self.ui.tablemodel)
 
     def add_ui_component(self):
         self.ui.searchComboBox.addItems([u'ALL',u'BOOKNAME',u'CONTENT',u'AUTHOR'])
@@ -101,16 +132,21 @@ class kmanWindow(QMainWindow):
     def refresh_ui_component(self, comp=0):
         """ refresh treeView, tableview, textedit information
             after import or open clips file
-        Args: comp 0 - treeview + tablevew + textedit
+        Args: comp 0 - treeview + tablevew + textedit , initial, fill_treeview()
-                   1 -            tablevew + textedit
+                                  move to __init__()
-                   2 -                       textedit
+                   1 -            tablevew + textedit , note tree view clicked
+                   2 -                       textedit , note table view clicked
+                   3 -            tablevew + textedit , word tree view clicked
+                   4 -                       textedit , word table view clicked
         """
-        if not comp in [1,2]:
+        # refresh tableview click note tree
-            self.fill_treeview()
+        if comp in [1,2]:
+            data = self.convert_to_panda(self.filter_list,0)
+        # refresh tableview click word tree
+        elif comp in [3,4]:
+            data = self.convert_to_panda(self.filter_list,1)
 
-        # refresh tableview
+        # refresh tableview content
-        if comp != 2:
-            data = self.convert_to_panda(self.filter_data)
         if hasattr(self.ui, 'tablemodel'):
             del self.ui.tablemodel
         self.ui.tablemodel = nTableModel(data)
@@ -119,23 +155,47 @@ class kmanWindow(QMainWindow):
         #self.ui.tablemodel.tabledata_update.connect(self.tabledata_update_slot)
 
         # refresh textedit content
-        if len(self.filter_data)>0:
+        if comp in [1,2]:
-            [stype,sbookname,sauthor,sposition,stime,scontent] = self.filter_data[0]
+            if len(self.filter_list)>0:
-            self.ui.textEdit.setHtml("""<span style='font-size:22pt;color:maroon'>《{bookname}》</span>
+                [stype,sbookname,sauthor,sposition,stime,scontent] = self.filter_list[0]
-            <span style='font-size:22pt;color:maroon'> {author} </span>
+                self.ui.textEdit.setHtml(notes_temp.format(
-            <span style='font-size:22pt;color:maroon'> ({time}) </span>
+                    bookname=sbookname, author=sauthor, time=stime,
-            <span style='font-size:11pt;color:maroon'> 【{note}】 </span><br><br>
-            <span><HR style="FILTER: progid:DXImageTransform.Microsoft.Shadow(color:#987cb9,direction:145,strength:15)" width="95%" color=#987cb9 SIZE=1></span>
-            <span style='font-size:22pt;color:#31849B'>{content}</span>
-            <span style='font-size:11pt;color:maroon'>【P{position}】</span><br>""".
-            format(bookname=sbookname, author=sauthor, time=stime,
                     note=stype, content=scontent, position=sposition))
+        elif comp in [3,4]:
+            self.render_textedit_words(self.words_data)
 
         self.show_status_info()
 
-    def convert_to_panda(self, seclist):
+    def render_textedit_words(self, wdata, mrow=100):
-        pdframe = pd.DataFrame(seclist, \
+        [bookinfo, words, lookups] = wdata
+
+        index = self.ui.tableView.currentIndex()
+        if index.row() == -1:
+            word = self.filter_list[0][2]
+        else:
+            word = index.sibling(index.row(), 2).data()
+
+        txt = ""
+        for row in lookups:
+            if words[row[1]]['word'] == word:
+                [susage, stimestamp, sbookname, sauthor, scategory, sposition] = \
+                        [row[4],row[5],bookinfo[row[2]]['bookname'], \
+                        bookinfo[row[2]]['author'], words[row[1]]['category'],row[3]]
+
+                txt += words_temp.format(
+                        usage=susage,bookname=sbookname,
+                        author=sauthor,category=scategory,
+                        timestamp=stimestamp,position=sposition)
+
+        self.ui.textEdit.setHtml(txt)
+
+    def convert_to_panda(self, mlist, tp=0):
+        if tp==0:
+            pdframe = pd.DataFrame(mlist, \
                     columns = ['Type','Bookname','Author','Position','Date','Content'])
+        else:
+            pdframe = pd.DataFrame(mlist, \
+                    columns = ['Bookname','Author','Word','Category'])
         return pdframe
 
     def tabledata_update_slot(self, s):
@@ -147,24 +207,50 @@ class kmanWindow(QMainWindow):
         itemmodel = model_index.model() #QAbstractItemModel/QStandardItemModel
         item = itemmodel.itemFromIndex(modelidx) #QStandardItem
 
-        description = item.accessibleDescription()
+        self.tree_selected = item.accessibleDescription()
-        print(description, item.text())
+        print(self.tree_selected, item.text())
 
         # filter clips
-        self.filter_data = None
+        self.filter_list = None
-        if description in ['root', 'bookname', 'author']:
-            [self.filter_books, self.filter_data] = self.km.filter_clips(self.books)
-        elif description == 'bleaf':  # bookname leaf
         info = re.split(r'\s+',item.text())[0]
-            [self.filter_books, self.filter_data] = self.km.filter_clips(self.books, info, 1)
+        comp = 0
-        else:  # author leaf
+        if self.tree_selected in ['note_root', 'note_bookname', 'note_author']:
-            info = re.split(r'\s+',item.text())[0]
+            [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data)
-            [self.filter_books, self.filter_data] = self.km.filter_clips(self.books, info, 2)
+            comp = 1
+        elif self.tree_selected in ['word_root', 'word_bookname']:
+            self.filter_list = self.km.filter_words(self.words_data)
+            comp = 3
+        elif self.tree_selected == 'note_bleaf':  # bookname leaf
+            comp = 1
+            [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data, info, 1)
+        elif self.tree_selected == 'note_aleaf':  # author leaf
+            comp = 1
+            [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data, info, 2)
+        elif self.tree_selected == 'word_leaf':  # word bookname leaf
+            comp = 3
+            self.filter_list = self.km.filter_words(self.words_data, info)
+        else: return
 
-        self.refresh_ui_component(comp=1)
+        if comp == 3:
+            self.ui.tableView.horizontalHeader().setStretchLastSection(True)
+            self.ui.tableView.setColumnWidth(0, 250) # author
+            self.ui.tableView.setColumnWidth(1, 50) # author
+            self.ui.tableView.setColumnWidth(3, 50) # category
+        if comp == 1:
+            self.ui.tableView.setColumnWidth(0, 40) # type
+            self.ui.tableView.setColumnWidth(2, 50) # author
+
+        self.refresh_ui_component(comp)
 
     def table_item_clicked(self, index):
-        if index.isValid():
+        """
+        print('tableView.currentIndex().row() {} tableView.currentIndex().column() {}'
+                .format(self.ui.tableView.currentIndex().row(),
+                    self.ui.tableView.currentIndex().column()))
+        """
+        if not index.isValid(): return
+
+        if self.tree_selected.split('_')[0]=='note':
             row = index.row()
             stype     = index.sibling(row, 0).data()
             sbookname = index.sibling(row, 1).data()
@@ -172,18 +258,12 @@ class kmanWindow(QMainWindow):
             sposition = index.sibling(row, 3).data()
             stime     = index.sibling(row, 4).data()
             scontent  = index.sibling(row, 5).data()
-        else:
-            return
 
-        self.ui.textEdit.setHtml("""<span style='font-size:22pt;color:maroon'>《{bookname}》</span>
+            self.ui.textEdit.setHtml(notes_temp.format(
-        <span style='font-size:22pt;color:maroon'> {author} </span>
+                bookname=sbookname, author=sauthor, time=stime,
-        <span style='font-size:22pt;color:maroon'> ({time}) </span>
-        <span style='font-size:11pt;color:maroon'> 【{note}】 </span><br><br>
-        <span>********************************************************</span><br><br>
-        <span style='font-size:22pt;color:#31849B'>{content}</span>
-        <span style='font-size:11pt;color:maroon'>【P{position}】</span><br>""".
-        format(bookname=sbookname, author=sauthor, time=stime,
                 note=stype, content=scontent, position=sposition))
+        else:
+            self.render_textedit_words(self.words_data)
 
     def search_button_clicked(self):
         search_word  = self.ui.searchLineEdit.text()
@@ -192,8 +272,8 @@ class kmanWindow(QMainWindow):
             return 0
         content_type = self.ui.searchComboBox.currentText()
         #content_idx  = self.ui.searchComboBox.currentIndex()
-        [nu, sbks] = self.km.search_clip(self.books, search_word, 'ALL', content_type)
+        [nu, sbks] = self.km.search_clip(self.books_data, search_word, 'ALL', content_type)
-        [self.filter_books, self.filter_data] = self.km.filter_clips(sbks)
+        [self.filter_books, self.filter_list] = self.km.filter_clips(sbks)
         self.refresh_ui_component()
 
         print( 'call search_button_clicked()' )
@@ -215,8 +295,8 @@ class kmanWindow(QMainWindow):
     def search_scope_change(self,t):
         p = {0:'ALL',1:'TITLE',2:'AUTHOR',3:'CONTENT'}
         s = self.ui.searchLineEdit.text()
-        #print(self.books)
+        #print(self.books_data)
-        #print(search_clip(self.books,s,'ALL',p[t]))
+        #print(search_clip(self.books_data,s,'ALL',p[t]))
         print('call search_scope_change()')
 
     ## XXX
@@ -250,8 +330,11 @@ class kmanWindow(QMainWindow):
             self.messagebox(ico=2, info='\n\n  kindle is not connected')
             return 0
 
-        self.books = self.km.import_clips(fp+'documents/'+KINDLEFN)
+        self.books_data = self.km.import_clips(fp+'documents/'+CLIPFN)
-        [self.filter_books, self.filter_data] = self.km.filter_clips(self.books)
+        self.words_data = self.km.import_words(fp+'system/vocabulary/'+WORDFN)
+        [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data)
+        self.filter_wordlist = self.km.filter_words(self.words_data)
+
         self.refresh_ui_component()
 
     def import_local(self):
@@ -261,8 +344,8 @@ class kmanWindow(QMainWindow):
                 "All Files (*);;Text Files (*.txt)")   # filter file type
         self.fn = fn
 
-        self.books = self.km.import_clips(fn)
+        self.books_data = self.km.import_clips(fn)
-        [self.filter_books, self.filter_data] = self.km.filter_clips(self.books)
+        [self.filter_books, self.filter_list] = self.km.filter_clips(self.books_data)
         self.refresh_ui_component()
 
         #print('filename ', fn, 'filetype ', ft)
@@ -270,22 +353,23 @@ class kmanWindow(QMainWindow):
 
     def fill_treeview(self):
         self.ui.model = QStandardItemModel()
+
         rootItem = self.ui.model.invisibleRootItem()
-        item = QStandardItem('All Notes ({})'.format(self.km.get_totalnum_nt(self.books)))
+        item = QStandardItem('All Notes ({})'.format(self.km.get_totalnum_nt(self.books_data)))
         icon = QIcon()
         icon.addFile(u":/icons/emblem_library.png", QSize(), QIcon.Normal, QIcon.Off)
         item.setIcon(icon)
-        item.setAccessibleDescription('root')
+        item.setAccessibleDescription('note_root')
         rootItem.appendRow(item)
         parent_item = item
 
         # add bookname tree
-        [numbooks, booknum] = self.km.get_bookname_num(self.books)
+        [numbooks, booknum] = self.km.get_bookname_num(self.books_data)
         bookname_item = QStandardItem('Bookname ({})'.format(numbooks))
         icon = QIcon()
         icon.addFile(u":/icons/book_open.png", QSize(), QIcon.Normal, QIcon.Off)
         bookname_item.setIcon(icon)
-        bookname_item.setAccessibleDescription('bookname')
+        bookname_item.setAccessibleDescription('note_bookname')
         parent_item.appendRow(bookname_item)
 
         if numbooks > 0:
@@ -294,16 +378,16 @@ class kmanWindow(QMainWindow):
                 icon = QIcon()
                 icon.addFile(u":/icons/book_open_bookmark.png", QSize(), QIcon.Normal, QIcon.Off)
                 item.setIcon(icon)
-                item.setAccessibleDescription('bleaf')
+                item.setAccessibleDescription('note_bleaf')
                 bookname_item.appendRow(item)
 
         # add author tree
-        [numauthor, authnum] = self.km.get_author_num(self.books)
+        [numauthor, authnum] = self.km.get_author_num(self.books_data)
         author_item = QStandardItem('Author ({})'.format(numauthor))
         icon = QIcon()
         icon.addFile(u":/icons/person.png", QSize(), QIcon.Normal, QIcon.Off)
         author_item.setIcon(icon)
-        author_item.setAccessibleDescription('author')
+        author_item.setAccessibleDescription('note_author')
         parent_item.appendRow(author_item)
         parent_item = author_item
 
@@ -313,9 +397,38 @@ class kmanWindow(QMainWindow):
                 icon = QIcon()
                 icon.addFile(u":/icons/user.png", QSize(), QIcon.Normal, QIcon.Off)
                 item.setIcon(icon)
-                item.setAccessibleDescription('aleaf')
+                item.setAccessibleDescription('note_aleaf')
                 author_item.appendRow(item)
 
+        # add words root
+        word_rootItem = self.ui.model.invisibleRootItem()
+        [numwords, wordnum] = self.km.get_book_word_num(self.words_data)
+        item = QStandardItem('All Words({})'.format(numwords))
+        icon = QIcon()
+        icon.addFile(u":/icons/emblem_library.png", QSize(), QIcon.Normal, QIcon.Off)
+        item.setIcon(icon)
+        item.setAccessibleDescription('word_root')
+        word_rootItem.appendRow(item)
+        word_parent_item = item
+
+        # add word bookname tree
+        word_bookname_item = QStandardItem('Bookname ({})'.format(numbooks))
+        icon = QIcon()
+        icon.addFile(u":/icons/book_open.png", QSize(), QIcon.Normal, QIcon.Off)
+        word_bookname_item.setIcon(icon)
+        word_bookname_item.setAccessibleDescription('word_bookname')
+        word_parent_item.appendRow(word_bookname_item)
+
+        print( [numwords, wordnum] )
+        if numwords > 0:
+            for k, v in wordnum.items():
+                item = QStandardItem('{} ({})'.format(k, v))
+                icon = QIcon()
+                icon.addFile(u":/icons/book_open_bookmark.png", QSize(), QIcon.Normal, QIcon.Off)
+                item.setIcon(icon)
+                item.setAccessibleDescription('word_leaf')
+                word_bookname_item.appendRow(item)
+
         self.ui.treeView.setModel(self.ui.model)
         self.ui.treeView.expandAll()
 
@@ -382,8 +495,10 @@ class kmanWindow(QMainWindow):
     # backup file when kman closed
     # read backup file when kman start
     def closeEvent(self, e):
-        with open(BACKUPFN, 'w', encoding='utf8', errors='ignore') as fw:
+        with open(BACKUPNOTEFN, 'w', encoding='utf8', errors='ignore') as fw:
-            fw.write(self.km.dict2json(self.books))
+            fw.write(self.km.dict2json(self.books_data))
+        with open(BACKUPWORDFN, 'w', encoding='utf8', errors='ignore') as fw:
+            fw.write(self.km.dict2json(self.words_data))
 
         # stop check thread
         self.flag = False

test_kman.py

@@ -8,11 +8,22 @@
 #########################################################
 
 import unittest
+import json
 
 from collections import defaultdict
 
 from kman import *
 
+# log info
+logger = logging.getLogger()
+#formatter = logging.Formatter
+#    ('%(asctime)s - %(pathname)s[line:%(lineno)d] - %(levelname)s: %(message)s')
+formatter = logging.Formatter('')
+handler = logging.FileHandler('debug')
+handler.setFormatter(formatter)
+logger.addHandler(handler)
+logger.setLevel(logging.DEBUG)
+
 class TestKman(unittest.TestCase):
     # initial
     def setUp(self):
@@ -175,32 +186,52 @@ class TestKman(unittest.TestCase):
         t_books = self.cre_tbooks()
         t_books_remove_nt = self.km.add_note_to_highlight(t_books)
         self.assertEqual((t_books_remove_nt['薛兆丰经济学讲义']['1']['content']).replace('\n',''),\
-                '边际就是“新增”带来的“新增”。'+NTPREF+ '山寨 假货 问题')
                 '边际就是“新增”带来的“新增”。'+NTPREF+ '山寨 假货 问题')
 
     def test_get_kindle_path(self):
         kp = self.km.get_kindle_path()
         s = u"kindle disconnected" if not kp else u"kindle connected {}".format(kp)
-
         print(s)
 
     def test_get_bookname_num(self):
         t_books = self.cre_tbooks()
         [nu, bn] = self.km.get_bookname_num(t_books)
         self.assertEqual(bn['薛兆丰经济学讲义'],2)
-    def test_get_author_num(self):
+
+    def test_get_author_num(self):
         t_books = self.cre_tbooks()
         [nu, bn] = self.km.get_author_num(t_books)
         self.assertEqual(bn['薛兆丰'],2)
 
-
+    def test_filter_clips(self):
+        t_books = self.cre_tbooks()
         # no filter
         bn = self.km.filter_clips(t_books, '薛兆丰', 0)
         logger.debug('========== 1 ==========\n')
         logger.debug(json.dumps(bn, indent=2, ensure_ascii=False))
-        print('filter 0 \n',bn)
+        bn = {}  
+
         # by bookname
         bn = self.km.filter_clips(t_books, '枪炮、病菌与钢铁 : 人类社会的命运 (世纪人文系列丛书·开放人文)', 1)
+        logger.debug('========== 2 ==========\n')
+        logger.debug(json.dumps(bn, indent=2, ensure_ascii=False))
+        bn = {}  
+
+        # by author
+        bn = self.km.filter_clips(t_books, '贾雷德·戴蒙德', 2)
+        logger.debug('========== 3 ==========\n')
+        logger.debug(json.dumps(bn, indent=2, ensure_ascii=False))
+        bn = {}  
+
+    # test import words & filter words
+    def test_import_words(self):
+        import pprint
+        [bookinfo, lookups, words] =  self.km.import_words(fp=(CLIPPATH+WORDFN))
+        # how to beauty print to logger file
+        logger.debug('========== 4 ==========\n')
+        logger.debug(json.dumps(bookinfo, indent=2, ensure_ascii=False))
+        logger.debug('========== 5 ==========\n')
+        logger.debug(json.dumps(lookups, indent=2, ensure_ascii=False))
         logger.debug('========== 6 ==========\n')
         logger.debug(json.dumps(words, indent=2, ensure_ascii=False))
 

ttranslator.py

@@ -0,0 +1,86 @@
+
+#coding=utf8
+import random
+import requests
+import hashlib
+appid = 'xxxxxx'
+secretKey = 'xxxxx'
+def get_md5(string):#返回字符串md5加密
+    hl = hashlib.md5()
+    hl.update(string.encode('utf-8'))
+    return hl.hexdigest()
+
+def en_to_zh(en_str):#英语翻译成中文
+    api_url = 'http://api.fanyi.baidu.com/api/trans/vip/translate'
+    salt = random.randint(32768,65536)
+    sign = get_md5(appid + en_str + str(salt) + secretKey)
+    api_data = {
+        'q':en_str,
+        'from':'en',
+        'to':'zh',
+        'appid':appid,
+        'salt':salt,
+        'sign':sign
+    }
+    req_get = requests.get(api_url,api_data)
+    result = req_get.json()
+    print(result)
+    return result['trans_result']
+
+print(en_to_zh('test'))
+
+'''
+import urllib.request
+import urllib.parse
+import json
+from tkinter import *
+root=Tk()
+root.title("翻译小程序")
+sw = root.winfo_screenwidth()
+#得到屏幕宽度
+sh = root.winfo_screenheight()
+#得到屏幕高度
+ww = 500
+wh = 300
+x = (sw-ww) / 2
+y = (sh-wh) / 2-50
+root.geometry("%dx%d+%d+%d" %(ww,wh,x,y))
+lb2=Label(root,text="输入英文翻译中文，或者输入中文翻译英文，按回车键翻译。--版权所有，翻录必究。")
+lb2.place(relx=0, rely=0.05)
+txt = Text(root,font=("宋体",20))
+txt.place(rely=0.6, relheight=0.4,relwidth=1)
+inp1 = Entry(root,font=("",20))
+inp1.place(relx=0, rely=0.2, relwidth=1, relheight=0.25)
+def run2(event):
+    txt.delete("0.0",END)
+    a = (inp1.get())
+    content = a
+    url='http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule'
+    data={}
+    data['i'] = content
+    data['from'] = 'AUTO'
+    data['to'] = 'AUTO'
+    data['smartresult'] = 'dict'
+    data['client'] = 'fanyideskweb'
+    data['salt'] = '15812376682056'
+    data['sign'] = 'a1246b257926af8432be022564ff79f5'
+    data['ts'] = '1581237668205'
+    data['bv'] = '656f750600466990f874a839d9f5ad23'
+    data['doctype'] = 'json'
+    data['version'] = '2.1'
+    data['keyfrom'] = 'fanyi.web'
+    data['action'] = 'FY_BY_CLICKBUTTION'
+    data = urllib.parse.urlencode(data).encode('utf-8')
+    response = urllib.request.urlopen(url,data)
+    html = response.read().decode('utf-8')
+    target = json.loads(html)
+    s=("%s"%(target['translateResult'][0][0]['tgt'])+"\n")
+    print(s)
+    txt.insert(END, s)
+def button1(event):
+    btn1 = Button(root, text='结果如下', font=("",12))
+    btn1.place(relx=0.35, rely=0.45, relwidth=0.2, relheight=0.15)
+inp1.bind("<Return>",run2)
+button1(1)
+root.mainloop()
+'''

vocab

@@ -0,0 +1,45 @@
+
+'''
+/Volumes/Kindle/system/vocabulary/vocab.db
+bookinfo = { 'book_key': {
+            'bookname':'bookname_xxx',
+            'authors':'author_xxx'},}
+words = { 'word_id': {
+            'word':'梅花鹿',
+            'category':0,
+            'lang':'zh' },}
+lookups = {'id':{'word_key':'zh:当',
+            'book_key':'iCR!8Z2VQD36EH0HN6FDJN9R8S1P94Z5',
+            'pos':'377816',
+            'usage':'xxxx',
+            'timestamp':'1571540228933'},}
+'''
+
+table: BOOK_INFO
+  id         - book key, CR!NYRVJ9BT315PKFRS9B9KVB0XF213:6F1EB741
+  asin       x
+  guid       x
+  lang       x zh/en
+  title      - bookname
+  authors    - author
+
+table: LOOKUPS
+  id         x lookups key, CR!NYRVJ9BT315PKFRS9B9KVB0XF213:6F1EB741:xxx
+  word_key   - zh:当
+  book_key   - book key, like CR!NYRVJ9BT315PKFRS9B9KVB0XF213:6F1EB741
+  dict_key   x
+  pos        - position, like AYkJAAAAAAAA:28781, 115907
+  usage      - usage
+  timestamp  - 1566738399454
+
+table: WORDS
+  id         - zh:当
+  word       x 当
+  stem       x
+  lang       - zh/en
+  category   - 0 learning, 100 mastered
+  timestamp  x 1591141963703
+  profileid  x
+
+word_id -> lookups_id
+