SHOW:
|
|
- or go back to the newest paste.
| 1 | import sqlite3 | |
| 2 | import argparse | |
| 3 | import json | |
| 4 | import glob | |
| 5 | ||
| 6 | ||
| 7 | conn = sqlite3.connect('galleries.db')
| |
| 8 | c = conn.cursor() | |
| 9 | ||
| 10 | ||
| 11 | def update_db(): | |
| 12 | c.execute('''
| |
| 13 | CREATE TABLE IF NOT EXISTS galleries | |
| 14 | ( | |
| 15 | id INTEGER PRIMARY KEY, | |
| 16 | type TEXT, | |
| 17 | artists TEXT, | |
| 18 | groups TEXT, | |
| 19 | parodys TEXT, | |
| 20 | tags TEXT, | |
| 21 | characters TEXT, | |
| 22 | language TEXT, | |
| 23 | galleryname TEXT, | |
| 24 | UNIQUE (id) | |
| 25 | ) | |
| 26 | ''') | |
| 27 | db_keys = ['id', 'type', 'a', 'g', 'p', 't', 'c', 'l', 'n'] | |
| 28 | files = glob.glob('galleries*.json')
| |
| 29 | for i, filename in enumerate(files): | |
| 30 | with open(filename, 'r', encoding='utf8') as f: | |
| 31 | data = json.load(f) | |
| 32 | print('({}/{}) reading file {} with {} entries'.format(i + 1, len(files), filename, len(data)))
| |
| 33 | for entry in data: | |
| 34 | current_obj = [entry[key] if key in entry else '' for key in db_keys] | |
| 35 | current_obj = ['|'.join(elem) if type(elem) is list else elem for elem in current_obj] | |
| 36 | c.execute("INSERT OR REPLACE INTO galleries VALUES (?,?,?,?,?,?,?,?,?)", current_obj)
| |
| 37 | conn.commit() | |
| 38 | c.execute('SELECT Count(*) FROM galleries')
| |
| 39 | num, = c.fetchone() | |
| 40 | print('DB has {} entries'.format(num))
| |
| 41 | ||
| 42 | ||
| 43 | def find(q): | |
| 44 | q = 'SELECT id, type, artists, groups, parodys, tags, characters, language, galleryname FROM galleries WHERE {0}'.format(q)
| |
| 45 | print(q) | |
| 46 | c.execute(q) | |
| 47 | return c | |
| 48 | ||
| 49 | ||
| 50 | def print_info(title, arr): | |
| 51 | if arr: | |
| 52 | arr = arr.split('|')
| |
| 53 | print('{}:'.format(title), ', '.join(arr))
| |
| 54 | ||
| 55 | ||
| 56 | def print_gallery(gallery_data): | |
| 57 | id, type, artists, groups, parodys, tags, characters, language, galleryname = gallery_data | |
| 58 | print('{} https://hitomi.la/galleries/{}.html'.format(type, id))
| |
| 59 | print_info('Title', galleryname)
| |
| 60 | print_info('Artists', artists)
| |
| 61 | print_info('Groups', groups)
| |
| 62 | print_info('Parodys', parodys)
| |
| 63 | print_info('Tags', tags)
| |
| 64 | print_info('Characters', characters)
| |
| 65 | print_info('Language', language)
| |
| 66 | ||
| 67 | ||
| 68 | def fetch(q): | |
| 69 | с = find(q) | |
| 70 | print('=' * 80)
| |
| 71 | n = 10 | |
| 72 | data = c.fetchmany(n) | |
| 73 | while len(data): | |
| 74 | for result in data: | |
| 75 | print_gallery(result) | |
| 76 | print('=' * 80)
| |
| 77 | data = c.fetchmany(n) | |
| 78 | if len(data) and input('Press Enter for more, type "q" to exit\n') is 'q':
| |
| 79 | break | |
| 80 | print('=' * 35, 'END', '=' * 40)
| |
| 81 | ||
| 82 | ||
| 83 | def format_query(field, vals=[]): | |
| 84 | if vals and len(vals): | |
| 85 | return ['{} NOT LIKE "%{}%"'.format(field, v[1:]) if v[0] == '!' else '{} LIKE "%{}%"'.format(field, v) for v in vals]
| |
| 86 | return [] | |
| 87 | ||
| 88 | ||
| 89 | def search_galleries(args): | |
| 90 | if args.id: | |
| 91 | fetch('id={}'.format(args.id))
| |
| 92 | else: | |
| 93 | q = [] | |
| 94 | ||
| 95 | if args.type != 'all': | |
| 96 | q.append('type="{}"'.format(args.type))
| |
| 97 | ||
| 98 | if args.language != 'all': | |
| 99 | q.append('language="{}"'.format(args.language))
| |
| 100 | ||
| 101 | if args.name: | |
| 102 | q.append('galleryname LIKE "%{}%"'.format(args.name))
| |
| 103 | ||
| 104 | q.extend(format_query('artists', args.artists))
| |
| 105 | q.extend(format_query('groups', args.groups))
| |
| 106 | q.extend(format_query('parodys', args.parodys))
| |
| 107 | q.extend(format_query('tags', args.tags))
| |
| 108 | q.extend(format_query('characters', args.characters))
| |
| 109 | ||
| 110 | if len(q): | |
| 111 | fetch(' AND '.join(q))
| |
| 112 | return True | |
| 113 | else: | |
| 114 | return False | |
| 115 | ||
| 116 | ||
| 117 | if __name__ == '__main__': | |
| 118 | parser = argparse.ArgumentParser(description='''Search for galleries in local copy of hitomi.la database. | |
| 119 | To use this script, first you have to manually download galleries*.json files from hitomi.la at urls: | |
| 120 | https://ltn.hitomi.la/galleries0.json, | |
| 121 | https://ltn.hitomi.la/galleries1.json, | |
| 122 | https://ltn.hitomi.la/galleries2.json, | |
| 123 | ..., | |
| 124 | https://ltn.hitomi.la/galleries19.json, etc. | |
| 125 | (there are currently 20 files, but there can be more in the future) | |
| 126 | Save then in the same folder as this script. | |
| 127 | Then, call "python hitomy.py update" to create SQLite database. After that, you can use "python hitomy.py search ..." for quick and simple search. | |
| 128 | To update galleries with new entries, re-download last galleriesXX.json (and add new JSONs if there are any) file and run "python hitomy.py update" again. | |
| 129 | You can delete all JSON files after update is done. | |
| 130 | ''', | |
| 131 | epilog='Use "!" in search queries for exclusion, i.e.: "search -t shota !yaoi"') | |
| 132 | ||
| 133 | parser.add_argument('action', type=str, choices=['search', 'update'],
| |
| 134 | help='update reads all galleries*.json files in directory and populates database, search will return results from query to database (see other options)') | |
| 135 | ||
| 136 | parser.add_argument('-id', type=int,
| |
| 137 | help='search by ID (other options are ignored)') | |
| 138 | ||
| 139 | parser.add_argument('--tags', '-t', metavar='T', type=str, nargs='+',
| |
| 140 | help='search tags') | |
| 141 | ||
| 142 | parser.add_argument('--artists', '-a', metavar='A', type=str, nargs='+',
| |
| 143 | help='search for artists') | |
| 144 | ||
| 145 | parser.add_argument('--groups', '-g', metavar='G', type=str, nargs='+',
| |
| 146 | help='search gallery groups') | |
| 147 | ||
| 148 | parser.add_argument('--parodys', '-p', metavar='P', type=str, nargs='+',
| |
| 149 | help='search parodys') | |
| 150 | ||
| 151 | parser.add_argument('--characters', '-c', metavar='C', type=str, nargs='+',
| |
| 152 | help='search for characters') | |
| 153 | ||
| 154 | parser.add_argument('--language', '-l', metavar='L', default='all', choices=['all', 'czech', 'italian', 'persian', 'polish', 'thai', 'esperanto', 'tagalog', 'ukrainian', 'danish', 'french', 'arabic', 'portuguese', 'romanian', 'english', 'korean', 'russian', 'vietnamese', 'slovak', 'hungarian', 'turkish', 'dutch', 'latin', 'spanish', 'greek', 'german', 'indonesian', 'norwegian', 'japanese', 'hebrew', 'catalan', 'finnish', 'albanian', 'estonian', 'chinese', 'swedish'],
| |
| 155 | help='search media language') | |
| 156 | ||
| 157 | parser.add_argument('--name', '-n', metavar='N', type=str,
| |
| 158 | help='search for title') | |
| 159 | ||
| 160 | parser.add_argument('--type', '-T', default='all', choices=['all', 'doujinshi', 'manga', 'artistcg', 'gamecg', 'anime'],
| |
| 161 | help='search media type') | |
| 162 | ||
| 163 | args = parser.parse_args() | |
| 164 | ||
| 165 | if args.action == 'search': | |
| 166 | r = search_galleries(args) | |
| 167 | if not r: | |
| 168 | parser.print_help() | |
| 169 | elif args.action == 'update': | |
| 170 | update_db() | |
| 171 | else: | |
| 172 | parser.print_help() | |
| 173 | ||
| 174 | conn.close() |