[Python] Qposts Research (v0.0.1b)

#!/usr/bin/env python3
# qposts_research.py
# version 0.0.1b (20200117)

# Below is a set of Python methods to facilitate research/analysis of Qposts.
# - needs Python (version 3.4+ );
# - uses only standard Python libraries;
# - tested only with Python 3.7.5 on Ubuntu 19.10
# - dictionary of Qpost abbreviations included.
#
# To run the script, type "python3 qposts_research.py" in your terminal (at the correct folder-location).
#  With this Python script you can:
#  - Download all Qposts as a single JSON-file from qanon.pub.
#  - Define arbitrary subsets of Qposts and print them out in any order.
#  - Perform a complex Text or Date Search using "and, or, and not"-operators.
#  - Perform a Regular Expression Search or group matching of Qpost data.
#  - Find Qclock-aligned dates, together with the Qposts posted on those dates.
#  - Download images linked in Qposts (100% success rate), including images from references (93% success).
#  - Export your Qposts JSON-file to an SQLite3 Database and perform SQL SELECT-queries from the terminal.
#
# Relevant Quotes from Q:
#    Q59:  "Combine all posts and analyze."
#    Q993: "Learn how to archive offline."
#    Q22:  "Study and prepare."
#
# This is Open-Source Freeware/QAnonware;
# Released As-Is; No Warranty; Use at Own Risk.
# May God Bless Q-team & QAnons Worldwide.
# WWG1WGA

import os
import re
import json
import html
import sqlite3
import argparse
import datetime
import dateutil.parser as dateparser
import xml.etree.ElementTree as xml_tree
from urllib.request import Request, urlopen
from urllib.error import URLError
from collections import Counter
from textwrap import wrap

_MAX_WRAP             = 120         # maximum wrap width for the Qposts "Text" field.
_LVL_INDENT           = ' ' * 4     # indented space for displaying nested references.
_INITIAL_SUBSET       = '*-1'       # Determines the initial subset of Qmap IDs (can be 'all', 'first N', 'last N', etc.).
_SHOW_ABBR_TOOLTIPS   = True        # True = Show Tooltips for known abbreviations in the Qposts "Text" field.
_QCLOCK_ROUND_NEAREST = True        # True = Qclock Hourhand Round-off to Nearest Minute; False = Round to Floor.
_VISIBLE_MENU_GROUPS  = {'A':1,'B':1,'C':1,'D':1} # Determines the initial visibility of each menu group (0=hidden; 1=visible).
                                                  # (Group D is only active if the user has an SQLite3 Qposts.db at _URL_QPOSTS_DB.)

_DT_FORMAT    = '%a %d %b %y'        # strftime format to display a short date string.
_DT_FORMAT_L  = '%A %d %B %Y'        # strftime format to display a long date string.
_DTM_FORMAT   = '%A %d %B %Y  %X %Z' # strftime format to display a long date&time string.

ok = '' # green color for Success.
er = '' # red color for Errors.
mc = '' # color for input Labels.
bl = ''             # background light blue for Qclock dates with at least 1 Qpost.
mf = ''                 # SQLite DB link color.
gr = ''                  # color for greyed-out text.
bw = ''                           # color for values: Bold white on black.
tc = ''                              # yellow for Qpost Labels + program text.
iv = ''   # invert
cu = ''   # italic
bd = ''   # bold
ts = ''   # reset to normal


_SAFE_URL_QPOSTS        = '~/Downloads/Qposts.json'     # location for the downloaded Qposts.json file.
_SAFE_URL_QPOSTS_DB     = '~/Downloads/Qposts.db'       # location for the Qposts SQLite-database.
_SAFE_URL_QPOSTS_IMAGES = '~/Downloads/Qposts_images/'  # location for downloaded Qpost images.

_URL_QPOSTS        = os.path.expanduser( _SAFE_URL_QPOSTS )     # un-anonymized versions of the url paths above.
_URL_QPOSTS_DB     = os.path.expanduser( _SAFE_URL_QPOSTS_DB )
_URL_QPOSTS_IMAGES = os.path.expanduser( _SAFE_URL_QPOSTS_IMAGES )

_QPOSTS        = []
_QMAP_IDS      = []
_QPOST_KEYS    = [ 'timestamp', 'text', 'media', 'references', 'name', 'trip', 'userId', 'link',
                   'source', 'threadId', 'id', 'title', 'subject', 'email', 'timestampDeletion' ]
                    # key timestampDeletion: used in 5 Qposts: [124,229,231,232,240] (all on 4plebs).
                    # title: used in 248 Qposts (on 4plebs/8chan_cbts); subject: used in all other Qposts (on 8ch/8kun).

_QPOST_DB_COLS = [ 'qmap_id' ]
_QPOST_DB_COLS.extend( _QPOST_KEYS )
_QPOST_DB_COLS.__setitem__( 4, 'refs' ) # NB. 'references' is a reserved word in SQLite3.

_DOC_STRFTIME    = 'https://docs.python.org/3/library/datetime.html#strftime-and-strptime-behavior'
_DOC_SQLITE_FUNC = 'https://www.sqlite.org/lang_corefunc.html'
_DOC_SQLITE_SLCT = 'https://www.sqlitetutorial.net/sqlite-select/'


# Dictionary of Qpost Abbreviations (Non-exhaustive; Collected from various sources):
# NB. this dictionary needs careful ordering of the items:
# - Keys that occur entirely inside another item's Value, must be placed BEFORE all such items (else the link formatting gets mixed up).
# - Keys that occur entirely inside another item's Key, must be placed AFTER all such items (else only the smaller Key will match).
#   In this way the smaller Keys will have their own tooltip inside the larger Key, and the larger Key will also have its own tooltip,
#   unless the larger Key consists entirely of smaller Keys...
_QPOST_ABBR  = {
    'AUS': 'Australia',
    'AZ': 'Arizona',
    'CA': 'California',
    'EU': 'European Union',
    'FL': 'Florida',
    'FR': 'France',
    'GER': 'Germany',
    'HI': 'Hawaii',
    'HK': 'Hong Kong',
    'HKG': 'Hong Kong',
    'LV': 'Las Vegas',
    'MX': 'Mexico',
    'NK': 'North Korea',
    'NY': 'New York',
    'NYC': 'New York City',
    'NZ': 'New Zealand',
    'PAK': 'Pakistan',
    'RUS': 'Russia',
    'SA': 'Saudi Arabia',
    'SK': 'South Korea',
    'TN': 'Tennessee',
    'TX': 'Texas',
    'UK': 'United Kingdom',
    'UN': 'United Nations',
    'US': 'United States',
    'USA': 'United States of America',
    'UT': 'Utah',
    'VA': 'Virginia',
    'WASH': 'Washington',
    'ATL': 'Atlanta Airport',
    'IAD': 'Dulles International Airport (Washington)',
    'PVG': 'Shanghai Pudong Airport',
    'AF1': 'Air Force 1 (Presidential Airplane)',
    'AG': 'Attorney General',
    'AMB': 'Ambassador',
    'CEO': 'Chief Executive Officer',
    'CEOs': 'Chief Executive Officers',
    'CIGIE': 'Council of Inspectors General on Integrity and Efficiency',
    'DAG': 'Deputy Attorney General',
    'IG': 'Inspector General',
    'NY_AG': 'Attorney General of New York State',
    'POTUS': 'President of the United States',
    'SCJ': 'Supreme Court Justice',
    'SD': 'State Department',
    'SIG': 'Signal; Special Interest Group',
    'SIT RM': 'Situation Room (White House)',
    'VP': 'Vice President',
    'WH': 'White House',
    'DoD': 'Department of Defense',
    'DoT': 'Department of Transportation',
    'DOJ': 'Department Of Justice',
    'DOE': 'Department Of Energy',
    'DHS': 'Department of Homeland Security',
    'DNC': 'Democratic National Committee',
    'A\'s': 'Agencies',
    'ABCs': 'Alphabet agencies (Acronyms of US governmental agencies)',
    'AFIA': 'Air Force Intelligence Agency',
    'AIA': 'Army Intelligence Agency',
    'CIA': 'Central Intelligence Agency',
    'C_A': 'CIA (Lacking Intelligence)',
    'DARPA': 'Defense Advanced Research Projects Agency',
    'DIA': 'Defense Intelligence Agency',
    'DNI': 'Director of National Intelligence',
    'ESC': 'Electronic Security Command (USAF)',
    'FBI': 'Federal Bureau of Investigation',
    'FISA': 'Foreign Intelligence Surveillance Act',
    'FISC': 'Foreign Intelligence Surveillance Court (FISA Court)',
    'FOIA': 'Freedom Of Information Act',
    'ICE': 'U.S. Immigration and Customs Enforcement',
    'INSCOM': 'United States Army Intelligence and Security Command',
    'IRS': 'Internal Revenue Agency',
    'ITAC': 'Intelligence and Threat Analysis Center (US Army)',
    'NASA': 'National Aeronautics and Space Administration',
    'NCTC': 'National Counter Terrorism Center',
    'NG': 'National Guard',
    'NOIC': 'Naval Operational Intelligence Center (US Navy)',
    'NPIC': 'National Photographic Intelligence Center (CIA)',
    'NSA': 'National Security Agency',
    'OCMC': 'Overhead Collection Management Center (NSA)',
    'OIG': 'Office of the Inspector General',
    'TSA': 'Transportation Security Administration',
    'SRD': 'Special Research Detachment (US Army)',
    'SS': 'Secret Service',
    'USSS': 'United States Secret Service',
    'FVEY': 'Five Eyes: an intelligence alliance comprising USA, UK, CAN, AUS, NZ',
    '5 Eyes': 'FVEY (an intelligence alliance comprising USA, UK, CAN, AUS, NZ)',
    'GCHQ Bude': 'UK Government satellite ground station and eavesdropping centre',
    'GCHQ': 'Government Communications Headquarters (UK)',
    'MI5': 'Military Intelligence Section 5 (UK Security Service)',
    'MI6': 'Military Intelligence Section 6 (UK Secret Intelligence Service)',
    'SIS': 'UK Secret Intelligence Service',
    'MOSSAD': 'Israeli Secret Intelligence Service',
    'MOS': 'MOSSAD (Israeli Secret Intelligence Service)',
    'MSM': 'Mainstream Media',
    'ARM': 'Anti-Republican Media',
    'AP': 'Associated Press',
    'AURN': 'American Urban Radio Networks',
    'ABC': 'American Broadcasting Company; Alphabet agencies',
    'BBC': 'British Broadcasting Corporation',
    'BUZZF': 'BuzzFeed',
    'CBS': 'Columbia Broadcasting System',
    'CNN': 'Cable News Network',
    'CNBC': 'Consumer News and Business Channel',
    'HuffPo': 'Huffington Post',
    'LAT': 'Los Angeles Times',
    'NBC': 'National Broadcasting Company',
    'MSNBC': 'US TV network partnership between Microsoft and NBC',
    'NPR': 'National Public Radio',
    'NYT': 'New York Times',
    'OANN': 'One America News Network',
    'PBS': 'Public Broadcasting Service',
    'WaPo': 'Washington Post',
    'WAPO': 'Washington Post',
    'WASHPOST': 'Washington Post',
    'WSJ': 'Wall Street Journal',
    'FB': 'Facebook',
    'GOOG': 'Google',
    'WL': 'WikiLeaks',
    'YT': 'YouTube',
    'JFK JR': 'John F. Kennedy Junior (son of President John F. Kennedy)',
    'JFK': 'John Fitzgerald Kennedy (35th US President); Gen. John Francis Kelly',
    'DJT': 'Donald John Trump (45th US President)',
    'Flynn JR': 'Michael Flynn Junior (son of General Flynn)',
    'GHWB': 'George Herbert Walker Bush (41st US President)',
    'GWB': 'George W. Bush (43rd US President)',
    'HRC': 'Hillary Rodham Clinton (Secretary of State in Obama\'s first term)',
    'Huma': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
    'IA': 'Information Assurance',
    'IC': 'Intelligence Community',
    'SC': 'Supreme Court; Special Counsel; Sara Carter (investigative reporter)',
    'Perkins Coie': 'DNC’s private law firm',
    'Fusion GPS': 'phony-intel firm was paid $1,024,408 by HRC/Perkins-Coie for creating the Steel Dossier',
    'Crowdstrike': 'CA-based Cyber-security company that falsely claimed that Russia had hacked the DNC servers',
    '\n+++': 'House of Saud',   # added \n for best result.
    '\n++': 'Rothschild family',
    '\n+': 'George Soros (Globalist billionaire)',
    '[]': 'kill box (area of interest)',
    '[R]': 'Renegade (Secret Service codename for Barack Obama); Rothschild (?)',
    '[E]': 'Eagle (Secret Service codename for Bill Clinton)',
    '[D]': 'Democrat; Democratic',
    '[D]s': 'Democrats',
    '1-800-273-8255': 'Phone number of the Veterans Crisis Line',
    '11.11.18.': 'IP address range of US DoD Network Information Center',
    '187': 'Police Code for homicide',
    '212-397-2255': 'Phone number of the Clinton Global Initiative',
    '302': 'FD-302 form used by the FBI for taking notes during an interview',
    '404': 'HTTP response code indicating "Page Not Found"',
    '4-10-20': 'letter values of initials DJT (Donald John Trump)',
    '4,10,20': 'letter values of initials DJT (Donald John Trump)',
    '4, 10, 20': 'letter values of initials DJT (Donald John Trump)',
    '4ch': '4chan (message board previously used by Q)',
    '5:5': 'Loud and Clear',
    '7th Floor': '"Shadow Government" within the SD that regularly met on the 7th floor of the Harry S. Truman Building in DC',
    '8ch': '8chan (message board previously used by Q)',
    '@JACK': 'Jack Dorsey (CEO of Twitter)',
    '@Jack': 'Jack Dorsey (CEO of Twitter)',
    '@Snowden': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
    '@SNOWDEN': 'Edward Snowden (CIA/NSA spy who leaked NSA documents)',
    'A Cooper': 'Anderson Cooper (CNN anchor, son of Gloria Vanderbilt)',
    'ADM R': 'Admiral Michael S. Rogers (Director of the NSA)',
    'Adm R': 'Admiral Michael S. Rogers (Director of the NSA)',
    'AJ': 'Alex Jones (Radio show host linked to Mossad)',
    'AL-Q': 'Al-Qaeda (Islamic terrorist group pursuing NATO\'s geostrategic goals)',
    'AL': 'Al Franken (Sen. D-MN)',
    'AM': 'Andrew McCabe (FBI Deputy Director); Ante Meridiem',
    'Anon': 'anonymous person',
    'ANTIFA': '"Anti-Fascists" (Soros backed fascists/domestic terrorists)',
    'AS': 'Adam Schiff (Rep. D-CA); Antonin Scalia (Supreme Court Associate Justice)',
    'ASF': 'American Special Forces (?); Administrative Support Facility (?)',
    'AW': 'Anthony Weiner (convicted pedophile ex-husband of Huma Abedin)',
    'AWAN': 'Imran Awan (DNC IT staffer who blackmailed House Members)',
    'B2': 'Stealth bomber; Bill Barr (US Attorney General under Trump)',
    'B/H C': 'Bill & Hillary Clinton',
    'BARR': 'Bill Barr (US Attorney General under GHWB and Trump)',
    'BB': 'Bill Barr (US Attorney General under GHWB and Trump)',
    'BC': 'Bill Clinton (42nd US President)',
    'BDT': 'Bulk Data Transfer; Blunt & Direct Time; Bangladeshi Taka (currency)',
    'BGA': 'Bundesverband Großhandel, Außenhandel (German trade association)',
    'BHO': 'Barack Hussein Obama (44th US President)',
    'BIDEN': 'Joseph Biden (VP under Obama)',
    'BO': 'Board Owner; Barack Obama; Bruce Ohr (Associate Deputy AG)',
    'BOD': 'Board of Directors',
    'BODs': 'Boards of Directors',
    'BP': 'Border Patrol; Bill Priestap (FBI Dep. Dir. of Counterintelligence under Obama and Trump)',
    'BRENNAN': 'John Brennan (23rd CIA Director)',
    'BS': 'Bernie Sanders (Sen. I-VT)',
    'CC': 'Chelsea Clinton (daughter of Bill & Hillary Clinton)',
    'CF': 'Clinton Foundation',
    'CFR': 'Council on Foreign Relations',
    'CHAI': 'Clinton Health Access Initiative',
    'C-Info': 'Confidential Information',
    'CLAPPER': 'Director of National Intelligence under Obama',
    'CLAS': 'Classification; Classified',
    'CLAS_OP_IAD_': 'Classified Operation at Dulles International Airport (?)',
    'Clowns In America': 'CIA',
    'CLOWNS IN AMERICA': 'CIA',
    'CM': 'CodeMonkey (8kun Admin); Cheryl Mills (Adviser to Hillary Clinton)',
    'CoC': 'Chain of Command; Chain of Custody',
    'COC': 'Chain Of Command; Chain Of Custody',
    'COMEY': 'James Comey (7th FBI Director)',
    'CORSI': 'Jerome Corsi, Mossad asset/agent',
    'CoS': 'Chief of Staff',
    'COV': 'Covert',
    'COVFEFE': 'Communications Over Various Feeds Electronically For Engagement Act',
    'CRUZ': 'Ted Cruz (Sen. R-TX)',
    'CS': 'Chuck Schumer (Sen. D-NY); Christopher Steele (former MI6); Civil Service',
    'D\'s': 'Democrats',
    'D’s': 'Democrats',
    'D+R+I': 'Democrat + Republican + Independent',
    'D5': 'Highest avalanche rating; December 5th; Chess move; 45=Trump',
    'DACA': 'Deferred Action for Childhood Arrivals (US immigration policy)',
    'DC': 'District of Columbia (Washington); Dan Coats (DNI under Trump); Dick Cheney (VP under G.W.Bush)',
    'DDoS': 'Directed Denial of Service (computer attack)',
    'DECLAS': 'Declassification; Declassified',
    'DEFCON': 'Defense Condition; Definitely Confirmed',
    'DF': 'Dianne Feinstein (Sen. D-CA)',
    'DL': 'Driver\'s License; David Laufman (Federal prosecutor); David Lawrence (Counsel to the Assistant AG)',
    'DM': 'Denis McDonough (White House Chief of Staff under Obama)',
    'DOA': 'Date Of Arrival; Dead Or Alive',
    'Donna': 'Donna Brazille (Hillary staffer)',
    'Dopey': 'Prince Al-Waleed bin Talal bin Abdulaziz al Saud',
    'DS': 'Deep State',
    'DWS': 'Debbie Wasserman Schulz (Rep. D-FL, DNC Chair under Obama)',
    'Eagle': 'Secret Service codename for President Bill Clinton',
    'Evergreen': 'Secret Service codename for Hillary Clinton',
    'EBS': 'Emergency Broadcast System',
    'EC': 'Eric Ciaramella (CIA agent)',
    'EG': 'Evergreen (Secret Service codename for Hillary Clinton)',
    'EH': 'Eric Holder (US Attorney General under Obama)',
    'EM': 'Emergency; Elon Musk (CEO of SpaceX and Tesla Inc.)',
    'EMP': 'Electromagnetic pulse',
    'EMS': 'Emergency Medical Services; Emergency Medical System',
    'EO': 'Executive Order',
    'EOs': 'Executive Orders',
    'EPSTEIN': 'Jeffrey Epstein (Billionaire who operated an elite pedophile ring for the Mossad)',
    'ES': 'Eric Schmidt (CEO of Google); Edward Snowden (CIA double agent who leaked NSA secrets)',
    'EST': 'Eastern Standard Time',
    'F + D': 'Foreign and Domestic',
    'F&F': 'Fast and Furious - Feinstein\'s failed gun sale attempt',
    'F2F': 'Face to Face',
    'f2f': 'face to face',
    'F9': 'Message Authentication Code integrity algorithm used by Facebook',
    'FED': 'Federal Reserve System (US Central Bank); Federal',
    'FEINSTEIN': 'Dianne Feinstein (Sen. D-CA)',
    'FF': 'False Flag',
    'FG&C': 'For God And Country',
    'FIRE & FURY': 'President Trump\'s warning to North Korea (8 Aug 2017)',
    'FISA_T_SURV': 'Targeted Surveillance authorized under Section 702 of the FISA Amendments Act',
    'FLYNN': 'Gen. Michael T. Flynn (National Security Advisor under Obama, fired for patriotism)',
    'FY': 'Fiscal Year',
    'G v E': 'Good versus Evil',
    'GA': 'Great Awakening',
    'GANG OF 8': 'Oversight board of the U.S. intelligence community',
    'GANG OF EIGHT': 'Oversight board of the U.S. intelligence community',
    'GDP': 'Gross Domestic Product',
    'GINA': 'Gina Haspel (25th CIA Director)',
    'GJ': 'Grand Jury',
    'GOODLATTE': 'Bob Goodlatte (Rep. R-VA)',
    'GOP': 'Grand Old Party (Republican Party)',
    'gov\'t': 'Government',
    'govt': 'Government',
    'Gov’t': 'Government',
    'Gov': 'Governor; Government',
    'GOV': 'Government',
    'GOWDY': 'Trey Gowdy (Rep. D-SC)',
    'GPS': 'Global Positioning System',
    'GRASSLEY': 'Chuck Grassley (Sen. R-IA)',
    'GS': 'George Soros (Billionaire globalist investor)',
    'GZ': 'Ground Zero',
    'HA': 'Huma Abedin (Personal Assistant to Hillary Clinton)',
    'HAM radio': 'Amateur radio',
    'HEC': 'House Ethics Committee',
    'HLS': 'Harvard Law School',
    'HOLDER': 'Eric Holder (US Attorney General under Obama)',
    'HOROWITZ': 'Michael Horowitz (DOJ Inspector General)',
    'HS': 'Homeland Security',
    'H-relief': 'Haiti earthquake relief effort coordinated by Bill Clinton',
    'HUBER': 'John Huber (US Attorney for Utah)',
    'HUMA': 'Harvard University Muslim Alumni; Huma Abedin',
    'HUMINT': 'Human Intelligence',
    'HUNTER': 'Hunter Biden (son of Joe Biden)',
    'HUSSEIN': 'Barack Hussein Obama (44th US President)',
    'HW': 'Hollywood',
    'HWOOD': 'Hollywood',
    'H wood': 'Hollywood',
    'H-wood': 'Hollywood',
    'ICBM': 'Inter-Continental Ballistic Missile',
    'ICIG': 'Inspector General of the Intelligence Community',
    'ISIS': 'Israeli Secret Intelligence Service; Islamic State in Iraq and Syria',
    'IW': 'Information Warfare',
    'IQT': 'In-Q-Tel (Private firm providing information technology to the CIA)',
    'James 8. Corney': 'Deliberate misspelling of "James B. Comey"',
    'JA': 'Julian Assange (Founder of Wikileaks)',
    'JB': 'John Brennan (CIA Director); Joe Biden (VP under Obama); Jim Baker (FBI General Counsel); Jeff Bezos (CEO of Amazon)',
    'JC': 'James Comey (FBI Director); James Clapper (DNI under Obama); John Carlin (Assistant AG); Josh Campbell (FBI Special Agent)',
    'JD': 'Jack Dorsey (CEO of Twitter)',
    'JK': 'John Kerry (Secretary of State under Obama), Jared Kushner (Senior Adviser under Trump)',
    'JL': 'John Legend (American singer/songwriter)',
    'John M': 'John McCain (Sen. R-AZ)',
    'JP': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
    'JR': 'Junior; Jim Rybicki (FBI Chief of Staff under Comey, fired by Wray)',
    'JS': 'Jeff Sessions (US Attorney General under Trump); John Solomon (investigative reporter)',
    'Judge K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
    'Justice K': 'Brett Kavanaugh (Supreme Court Associate Justice)',
    'KAV': 'Brett Kavanaugh (Supreme Court Associate Justice)',
    'KC': 'Kevin Clinesmith (FBI attorney)',
    'KKK': 'Klu Klux Klan (created by the Democrats)',
    'KM': 'Kelly Magsamen (Special Assistant to the President)',
    'LARP': 'Live Action Role Player',
    'LifeLog': 'Pentagon DARPA mass-surveillance project rebranded as Facebook',
    'LdR': '(Lady) Lynn Forester de Rothschild (married to Evelyn de Rothschild)',
    'LDR': '(Lady) Lynn Forester de Rothschild',
    'LL': 'Loretta Lynch (US Attorney General under Obama)',
    'LLC': 'Limited Liability Company',
    'LP': 'Lisa Page (FBI Special Counsel)',
    'LYNCH': 'Loretta Lynch (US Attorney General under Obama)',
    'M’s': 'Marines',
    'Maxine W': 'Maxine Waters (Rep. D-CA)',
    'Mc_I': 'John McCain Institute',
    'MACRON': 'Emmanuel Macron (President of France)',
    'MAGA': 'Make America Great Again',
    'MAY': 'Theresa May (Prime Minster of UK)',
    'MB': 'Muslim Brotherhood',
    'MCCABE': 'Andrew McCabe (FBI Deputy Director under Comey, fired)',
    'MERKEL': 'Angela Merkel (Chancellor of Germany)',
    'MI': 'Military Intelligence',
    'MIL': 'Military',
    'MK': 'Mike Kortan (FBI Assistant Director)',
    'ML': 'Martial Law',
    'MLK': 'Martin Luther King (Civil rights advocate murdered in 1968)',
    'MM': 'Mary McCord (Principal Deputy Assistant AG); Media Matters',
    'MO': 'Michelle Obama (transvestite husband of Barack Obama)',
    'MOAB': 'Mother Of All Bombs',
    'MS': 'Microsoft; Michael Steinbach (FBI Executive Assistant Director)',
    'MS13': 'Latino Drug Cartel; MSM',
    'MUELLER': 'Robert Mueller (6th FBI Director)',
    'MURKOWSKI': 'Lisa Murkowski (Sen. R-AK)',
    'MW': 'Maxine Waters (Rep. D-CA)',
    'MZ': 'Mark Zuckerberg (CEO of Facebook)',
    'N&S': 'North and South',
    'N1LB': 'No One Left Behind',
    'No Name': 'John McCain',
    'No Such Agency': 'NSA',
    'NAT': 'National',
    'NATSEC': 'National Security',
    'NOFORN': 'No Foreign Nationals (Document Sensitivity Level)',
    'NO NAME': 'John McCain',
    'NO SUCH AGENCY': 'NSA',
    'NP': 'Nancy Pelosi (Rep. D-CA); Non-Profit',
    'NPO': 'Non-Profit Organization',
    'NR': 'Nuclear Reactor; Nuclear Radiation',
    'NSC': 'National Security Council',
    'NUNES': 'Devin Nunes (Rep. R-CA)',
    'NWO': 'New World Order; Nazi World Order (?)',
    'NXIVM': 'Sex trafficking cult with close ties to Democratic Party',
    'OO': 'Oval Office (White House)',
    'OP': 'Operation; Operator; Original Post; Original Poster; Operated Plane',
    'OPs': 'Operations',
    'OS': 'Oversight',
    'OWL': 'Orbital Weapon Lancet (Space-based weapon) (?)',
    'P': 'POTUS; Presidential; Pope; Peninsula; Paragraph; Page; Payseur',
    'P_Pers': 'POTUS Personal',
    'PAC': 'Political Action Committee',
    'PAGE': 'Lisa Page (FBI Special Counsel)',
    'PANIC': 'Patriots Are Now In Control',
    'PD': 'Police Department',
    'PDB': 'President’s Daily Brief',
    'PDBs': 'President’s Daily Briefs',
    'PELOSI': 'Nancy Pelosi (Rep. D-CA)',
    'PENCE': 'Mike Pence (VP under Trump)',
    'PEOC': 'Presidential Emergency Operations Center',
    'PG': 'Pizzagate/Pedogate',
    'PL': 'Presidential Library',
    'PM': 'Prime Minister; Post Meridiem; Paul Manafort (Campaign manager for Trump)',
    'PODESTA': 'John Podesta (WH Chief of Staff under Clinton, Counselor under Obama)',
    'POS': 'Piece Of Shit',
    'POV': 'Point Of View',
    'POVs': 'Points Of View',
    'PP': 'Planned Parenthood',
    'PRISM': 'NSA Internet data collection program',
    'PS': 'Peter Strzok (FBI Lead Agent); PlayStation',
    'PST': 'Pacific Standard Time',
    'PTSD': 'Post-Traumatic Stress Disorder',
    'PUTIN': 'Vladimir Putin (President of Russia)',
    'Q&A': 'Question and Answer',
    'Q+': 'President Trump; Q-team',
    'R v W': 'Right versus Wrong',
    'R\'s': 'Republicans',
    'R’s': 'Republicans',
    'R+D': 'Republican + Democrat',
    'RB': 'Rachel Brand (Associate AG)',
    'RBG': 'Ruth Bader Ginsburg (Supreme Court Associate Justice)',
    'RC': 'Rachel Chandler (Child handler for Jeffrey Epstein who did not kill himself)',
    'RE': 'Rahm Emanuel (White House Chief of Staff under Obama)',
    'RED RED': 'Red Cross',
    'RED_RED': 'Red Cross',
    'RENEGADE': 'Secret Service codename for Barack Hussein Obama',
    'RICE': 'Susan Rice (National Security Advisor under Obama)',
    'RIP': 'Rest In Peace',
    'RM': 'Robert Mueller (6th FBI Director)',
    'RNC': 'Republican National Committee',
    'RR': 'Rod Rosenstein (Deputy Attorney General under Trump)',
    'RT': 'Real Time; Retweet; Rex Tillerson (Secretary of State under Trump)',
    'RUDY': 'Rudy Giuliani (former Mayor of NYC)',
    'SB': 'Senate Bill',
    'SAP': 'Special Access Program',
    'SAPs': 'Special Access Programs',
    'Scaramucci': 'Anthony Scaramucci (WH Communications Director under Trump, fired after repeated TDS-attacks on Trump)',
    'SCHUMER': 'Chuck Schumer (Sen. D-NY)',
    'SCI': 'Sensitive Compartmented Information (TOP SECRET+)',
    'SCIF': 'Sensitive Compartmented Information Facility',
    'SDNY': 'Southern District of New York',
    'SEC': 'Security; Section',
    'SEC_TEST': 'Security Test',
    'SESSIONS': 'Jeff Sessions (US Attorney General under Trump)',
    'SH': 'Sean Hannity (Conservative TV host); Steve Huffman (CEO of Reddit)',
    'SIGINT': 'Signals Intelligence',
    'SIT ROOM': 'Situation Room (White House)',
    'SM': 'Sally Moyer',
    'SMOLLETT': 'Jussie Smollett (Hollywood actor who faked his own lynching)',
    'SOROS': 'George Soros (Billionaire globalist investor)',
    'SOTU': 'State Of The Union',
    'SP': 'Samantha Power (US Ambassador to the UN)',
    'SR': 'Seth Rich (DNC staffer murdered after leaking to Wikileaks); Susan Rice (National Security Advisor under Obama); Senior',
    'ST': 'Shit (?)',
    'STEELE': 'Christopher Steele (MI6 agent who concocted the Steele-dossier)',
    'STRAT': 'Strategic',
    'STRZOK': 'Peter Strzok (FBI agent who participated in the attempted subversion of the 2016 presidential election)',
    'SURV': 'Surveillance',
    'SY': 'Sally Yates (Deputy Attorney General)',
    'TBA': 'To Be Announced',
    'TG': 'Trey Gowdy (Rep. D-SC); Tashina Gauhar (FISA lawyer)',
    'TM': 'Team',
    'TP': 'Tony Podesta (Brother of John Podesta)',
    'TRI': 'Trilateral Commission (?)',
    'TT': 'Trump Tower; Tarmac Tapes',
    'T-Tower': 'Trump Tower',
    'U1': 'Uranium One',
    'UBL': 'Usama Bin Laden',
    'UC': 'Univerity of California',
    'USD': 'US Dollar',
    'USMIL': 'US Military',
    'VIP': 'Very Important Person',
    'VIPs': 'Very Important Persons',
    'VJ': 'Valerie Jarret (Senior Advisor to Obama)',
    'W&W': 'Wizards and Warlocks',
    'WHITAKER': 'Matthew G. Whitaker (Acting US Attorney General after Sessions resigned)',
    'WIA': 'Wounded In Action',
    'WMDs': 'Weapons of Mass Destruction',
    'WRAY': 'Christopher Wray (8th FBI Director)',
    'WRWY': 'We Are With You',
    'WW': 'World Wide; World War',
    'WWI': 'World War 1',
    'WWII': 'World War 2',
    'WWIII': 'World War 3',
    'WWG1WGA': 'Where We Go One We Go All',
    'XKeyscore': 'NSA Internet data search and analysis tool'
}

#################
# Methods for Qposts Research:

def qmap_id_to_qpost_index( qmap_id ):
    '''<qmap_id> can either be a Qmap ID, or a tuple/list of Qmap IDs.
    Qmap IDs are 1-based and run from oldest to latest, while Qpost index numbers are 0-based and run from latest to oldest.'''
    if isinstance( qmap_id, ( tuple, list ) ): return [ len( _QPOSTS ) - idx for idx in qmap_id ]
    return len( _QPOSTS ) - qmap_id


def qpost_index_to_qmap_id( qpost_idx ):
    '''For clarity; Same output as qmap_id_to_qpost_index( qpost_idx ).'''
    return qmap_id_to_qpost_index( qpost_idx )


def is_valid_qmap_id( qmap_id ):
    '''Returns True if <qmap_id> is a valid Qmap ID number.'''
    return qmap_id > 0 and qmap_id <= len( _QPOSTS )


def open_qposts_json( url_qposts_json ):
    '''Loads the specified JSON-file, and returns a list of dictionaries, or None.'''
    safe_url = collapse_user( url_qposts_json ) # Remove user name before printing.
    if os.path.exists( url_qposts_json ):
        try:
            with open( url_qposts_json, 'r', encoding='utf-8' ) as qps_file:
                return json.loads( qps_file.read() )
        except: print( f"JSON: Failed to load Qdata from file '{safe_url}'." )
    else: print( f"Error: No such file: '{safe_url}'." )


def download_qposts_json( url_save ):
    '''Downloads the Qposts JSON-file from qanon.pub, and saves it to the specified url.
    This function returns a list of dictionaries taken from the downloaded JSON-file, or None.'''
    _URL_JSON = "https://qanon.pub/data/json/posts.json"
    try:    # DOWNLOAD JSON DATA FROM QANON.PUB.
        req = Request( _URL_JSON, headers={'User-Agent': 'Mozilla/5.0'} )
        qdata = urlopen( req )
    except Exception as e: print( f'{er}Failed to download json file "{_URL_JSON}":{ts} {e}' ); return None
    try: qposts = json.loads( qdata.read().decode() )
    except Exception as e: print( f'{er}JSON: Failed to load Qpost data from downloaded file;{ts} {e}' ); return None
    safe_url = collapse_user( url_save ) # Hide user name before printing.
    try:    # DUMP DATA TO JSON FILE.
        with open( url_save, "w", encoding="utf-8" ) as qps_file:
            json.dump( qposts, qps_file, ensure_ascii=False, indent=2 )
    except Exception as e: print( f'{er}JSON: Failed to save Qpost data to file "{safe_url}":{ts} {e}' ); return None
    return qposts


def download_latest_qpost_nr():
    '''Downloads a (limited) XML RSS-file from qalerts.app.
    Returns a 2-tuple with the number of the most recent Qpost from qalerts.app, plus the root node to an XML ElementTree
    containing (limited) records of the 20 most recent Qposts from qalerts.app; Returns (None,None) if something went wrong.
    NB. The latest Qpost on qalerts.app is not necessarily available at exactly the same moment as the qposts.json at qanon.pub.'''
    # import xml.etree.ElementTree as xml_tree
    try:   # DOWNLOAD RSS-DATA FROM QALERTS.APP
        _URL_RSS = "https://qalerts.app/data/rss/posts.rss"
        req = Request( _URL_RSS, headers={ 'User-Agent': 'Mozilla/5.0' } )
        qdata = urlopen( req )
    except Exception: return None, None
    root = xml_tree.fromstring( qdata.read().decode() )
    try:   # PARSE RSS-DATA
        text = root[0][10][0].text
        return int( text.replace( 'Q Drop #', '' ) ), root
    except: return None, root


def check_update_local_qposts():
    '''This function checks online if there are new Qposts available, and if so, it offers to download them.
    The local Qposts.json file will be overwritten; if there exists a local Qposts.db database file,
    it will be updated by adding only new records for the new Qposts.
    Returns the updated current number of Qposts inside the local Qposts.json file.'''
    global _QPOSTS
    n_qposts = len( _QPOSTS )
    l_qpost, rss_root = download_latest_qpost_nr()  # Check for latest Qposts online.
    if l_qpost:
        if l_qpost > n_qposts:                      # NEW Qpost(s) available!
            n_new = l_qpost - n_qposts
            answer = input( f"{tc}🔔{ts+tc} There {'is' if n_new == 1 else 'are'} {bw} {n_new} {ts+tc} New Qpost" +
            f"{'' if n_new == 1 else 's'} available!{ts}\n{mc}Do you want to update your local Qposts now? (Y/n):{ts} " )
            if answer.lower() not in ['n', 'no']:   # Update Qposts.json, and Qposts.db if present.
                print( f"{tc}Updating local Qposts:{ts} Attempting to download Qposts.json from qanon.pub ..." )
                _QPOSTS = download_qposts_json( _URL_QPOSTS )
                n_qposts = len( _QPOSTS )
                if os.path.exists( _URL_QPOSTS_DB ): # Update Qposts.db if present.
                    qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
                    n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
        elif l_qpost == n_qposts:
            print( f'{ok}Online check completed: Your local Qposts.json is already up to date.{ts}' )
    else: print( f'{er}Could not check online if there are any new Qposts available.{ts}' )
    return n_qposts


def get_qpost_media_urls( qpost ):
    '''Returns a nested list of tuples(url,filename) for all media linked in <qpost> and in its references recursively.'''
    tuples = []
    if isinstance( qpost, dict ):
        qpost_media = qpost.get( 'media', [] )
        if qpost_media:
            for image in qpost_media:
                tuples.append( ( image.get( 'url', '' ), image.get( 'filename', '' ) ) )
        qpost_refs  = qpost.get( 'references', [] )     # this tag is only present if the Qpost has references.
        if qpost_refs:
            for qp_ref in qpost_refs:
                tuples.append( get_qpost_media_urls( qp_ref ) )
    return tuples


def download_qpost_images( qmap_ids_list, references_too=False ):
    '''Workaround; Tries to download the media linked in all Qposts whose Qmap ID is specified in <qmap_ids_list>;
    This creates a subfolder for each Qmap ID that has linked media, and saves all downloaded media for that Qmap ID inside that subfolder.
    Subfolders for downloaded media will all be created inside the folder <_URL_QPOSTS_IMAGES>, and will be named after the Qmap ID number.
    Qposts can contain references, that can in turn also contain media. To recursively download these media too, pass <references_too>=True.
    NB. this could result in the downloading of multiple duplicate image files, when a Qpost with images is referenced in another Qpost.
    Statistics for 3774 Qposts:
    excl. References: total  809 media;  694 subfolders; DL size ~370MB; DL time ~~20min; DL success-rate 100%.
    incl. References: total 1941 media; 1273 subfolders; DL size ~864MB; DL time ~~50min; DL success-rate ~93% (Fails for 136 files).'''
    # import os
    def download_urls( url_list, save_folder, references_too ):
        '''Download files from nested list <url_list> and save them into the folder <save_folder>.'''
        urls_to_replace  = [ 'https://media.8ch.net/file_store/thumb/',
                             'https://media.8ch.net/file_store/',
                             '//media.jthnx5wyvjvzsxtu.onion/file_store/' ]
        url_replacements = [ 'https://qalerts.app/media/',
                             'https://qposts.online/assets/images/' ]
        total_media, dl_success, already_present = 0, 0, 0
        for item in url_list:
            nm, ns, ap = 0, 0, 0
            if isinstance( item, tuple ) and len(item) == 2:
                total_media += 1
                image_url, filename = item
                image_url_rep = image_url
                filename2 = os.path.split( image_url )[1]   # storage name.
                if not filename: filename = filename2
                save_url = os.path.join( save_folder, filename )
                if not os.path.exists( save_url ):

                    # Added since 8chan is offline, making all 8ch.net image-urls invalid;
                    # instead try to download the corresponding images from qposts.online or qalerts.app.
                    # TODO: change this part when 8kun has transfered all the older images from 8ch.net.
                    for url_replacement in url_replacements:
                        for url in urls_to_replace:
                            if image_url.startswith( url ):
                                image_url_rep = image_url.replace( url, url_replacement ); break
                        save_ok  = download_binary_file( image_url_rep, save_url )     # Download/Save media.
                        if not save_ok:                                                # Download failed:
                            fparent, fname = os.path.split( image_url_rep )            # Retry with filename2.
                            image_url_rep  = os.path.join( fparent, filename2 )
                            save_ok  = download_binary_file( image_url_rep, save_url )
                        if save_ok: break

                    dl_success += bool( save_ok )
                    url_safe = collapse_user( save_url )
                    if save_ok: print( f"{ok}Success:{ts} media '{image_url_rep}'\n\t{ok} was saved{ts} to file '{url_safe}'." )
                    else: print( f"{er}Failed:{ts} media '{image_url_rep}'\n\t{er} was not saved{ts} to file '{url_safe}'." )
                else: already_present += 1
            elif isinstance( item, list ) and references_too:
                nm, ns, ap = download_urls( item, save_folder, references_too )
            total_media += nm; dl_success += ns; already_present += ap
        return total_media, dl_success, already_present
        ###### END of download_urls().

    if not os.path.exists( _URL_QPOSTS_IMAGES ): os.mkdir( _URL_QPOSTS_IMAGES )
    tuples  = get_qposts_by_id( qmap_ids_list, key='', qmap_ids=True )
    total_qposts, total_media, qp_has_media, success, already_present = 0, 0, 0, 0, 0
    for qmap_id, qpost in tuples:
        total_qposts += 1
        qpost_media = get_qpost_media_urls( qpost )    # Nested list of tuples(url,filename).
        qpost_own_media, qpost_ref_media = split_list( qpost_media )
        if qpost_own_media or qpost_ref_media:
            qp_has_media += 1
            folder_qmap_id = os.path.join( _URL_QPOSTS_IMAGES, f'{qmap_id}', '')
            if not os.path.exists( folder_qmap_id ):
                if references_too or qpost_own_media: os.mkdir( folder_qmap_id )        # Create a subfolder for this QmapID.
            nm, ns, ap = download_urls( qpost_media, folder_qmap_id, references_too )   # Download Qpost media into subfolder.
            total_media += nm; success += ns; already_present += ap
    n_failed = total_media - already_present - success
    print( f"{tc}Processed {bw} {total_qposts} {tc} Qposts ({['excl.','incl.'][references_too]} references);{ts}\n{bw} {qp_has_media} {tc} " +
           f"Qposts contain a total of {bw} {total_media} {tc} linked media:{ts}\n{tc}Media Present:{ts} {already_present}/{total_media}" +
           f"{tc}   Media Downloaded: {ok} {success}/{total_media} {tc}   Download Failed: {er} {n_failed}/{total_media} {tc}.{ts}" )


def get_qposts_by_id( qpost_ids, key='', qmap_ids=False ):
    '''Returns a list with tuples(qmap_id,element) of all qposts whose indexes are specified in <qpost_ids>.
    <qpost_ids>: List of indexes into the _QPOSTS list ( or list of Qmap IDs if <qmap_ids>=True ).
    <key>: if given, the returned list elements will be only the field qpost[key]; else they will be the entire qpost dict.
    <qmap_ids>: if True, the given <qpost_ids> are interpreted as Qmap IDs ranging from 1 to N ( where 1 is the first Qpost),
    else they are interpreted as indexes into the _QPOSTS list, ranging from 0 to N-1 ( where 0 is the latest Qpost).'''
    if _QPOSTS:
        qp_count = len( _QPOSTS )
        result  = []
        for i in qpost_ids:
            qmap_id = i if qmap_ids else qp_count - i
            index   = qp_count - i if qmap_ids else i
            qpost   = _QPOSTS[index]
            if key: result.append( ( qmap_id, qpost.get(key) ) )
            else: result.append( ( qmap_id, qpost ) )
        return result


def search_qposts_regexp( regexp, key='' ):
    '''Returns a list with tuples(qmap_id,qpost) of all qpost dicts whose <key>-field matches <regexp>.
    <regexp>: String representing the regular expression to match.
    <key>: Pass a qpost dict key, or pass "" to search the entire qpost dict as string.
    Valid keys: see _QPOST_KEYS.'''
    # import re
    result = []
    for i, qpost in enumerate( _QPOSTS ):
        find_in = qpost[key] if qpost.get( key ) else str(qpost)
        if re.search( regexp, find_in ): result.append( ( qpost_index_to_qmap_id(i), qpost ) )
    return result


def search_qposts( find, key='text' ):
    '''Returns a list with tuples(qmap_id,qpost) of all qposts whose <key>-field contains <find>.
    <find>: can be a complex search term using operators (and, or, not) and parentheses; complex atoms must be double-quoted.
    If <key> is empty, it searches the text of the whole qpost dictionary for <find>; Some keys may not be present in all qposts.
    Valid keys: see _QPOST_KEYS.'''
    result  = []
    x = Simple_Logic_Expression( find )   # class included below.
    for i, qpost in enumerate(_QPOSTS):
        text = qpost.get( key, '' ) if key else qpost
        if not isinstance( text, str ): text = str( text )
        if text and x.find_in( text ):
            result.append( ( qpost_index_to_qmap_id(i), qpost ) )
    return result


def search_qposts_by_date( date_condition ):
    '''Returns a list with tuples(Qmap_id, Qpost) of all qposts whose timestamp matches <date_condition>.
       <date_condition>: String of the format "COMP DATETIME", where COMP is one of the comparison operators in:
       [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ], and DATETIME is a valid datetime expression such as '1 Aug 2019',
       or a timestamp number. Each "COMP DATETIME" pair must be enclosed within double quotation marks,
       and multiple "COMP DATETIME" pairs can be combined using logical connectives [and,or] and parentheses.
       for example: ("> 1 Aug 2019" and "< 2 Aug 2019") or "> 1 Jan 2020" .'''
    result  = []
    x = Simple_Logic_Expression( date_condition )   # class included below.
    for i, qpost in enumerate(_QPOSTS):
        timestamp = qpost.get( 'timestamp', -1 )
        if not isinstance( timestamp, (int,float) ): timestamp = float( timestamp )
        if timestamp and x.match_value( timestamp, evaluate_datetime_condition ):   # pass function as arg.
            result.append( ( qpost_index_to_qmap_id(i), qpost ) )
    return result


def get_qposts_for_date( d ):
    '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on the specified day <d>.
       <d>: datetime.datetime object representing the date for which to return all Qposts that were posted on the same day.'''
    return search_qposts_by_date( d.timestamp() )


def get_qposts_for_dates( dates ):
    '''Returns a list of tuples(QmapID, Qpost) whose Qpost timestamp falls on any of the dates specified in <dates>.
       <dates>: list of datetime.datetime objects representing the dates for which to return all Qposts.'''
    result  = []
    for d in dates:
        result.extend( get_qposts_for_date( d ) )
    return result


def get_qpost_dates_for_qmap_ids( qmap_ids_list ):
    '''Returns a list of datetime objects representing the posting dates of the Qposts specified in <qmap_ids_list>.'''
    qposts = get_qposts_by_id( qmap_ids_list, key='timestamp', qmap_ids=True )
    return [ datetime.datetime.fromtimestamp( float( qp[1] ), tz=None ) for qp in qposts ]


def qclock_get_aligned_dates_for_date( qdate=datetime.datetime.today(), include_mirror=True ):
    '''Collect earlier dates from the Qclock, that are located on the same radius or diameter line as the date specified in <qdate>.
       <qdate>: a datetime.datetime object or a string representing the date for which to retrieve the earlier Qclock-aligned dates;
       When passing a date string, prevent ambiguities by putting the day number before the month, and passing a 4-digit year.
       <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
    Returns a tuple with 4 elements: (the parsed input date; a standard string representation of the parsed input date; a list containing
    the aligned dates on this side of the center; and a list containing the aligned dates on the opposite side of the center: this latter
    list will be empty if <include_mirror>=False).'''
    # import datetime; _DT_FORMAT_L = '%A %d %B %Y'
    if isinstance( qdate, str ): dt, dts, _ = parse_datetime_string( qdate, _DT_FORMAT_L )
    elif isinstance( qdate, datetime.datetime ): dt, dts = qdate, qdate.strftime( _DT_FORMAT_L )
    if not dt: return None, '', [], []
    else: d_target = dt.toordinal()   # Serial Day Number of input date.
    aligned, aligned_mirror, current, mirror = [], [], d_target, d_target - 30
    d_start = 736630   # QClock Start: '10-28-17'; ( hour, angle, minutes ) = ( 4, 10, 20 ) = DJT.
    while current >= d_start:
        aligned.append( datetime.datetime.fromordinal( current ) ); current -= 60
    if include_mirror:
        while mirror >= d_start:
            aligned_mirror.append( datetime.datetime.fromordinal( mirror ) ); mirror -= 60
    return dt, dts, aligned, aligned_mirror


def qclock_get_aligned_dates_for_clocktime( qtime=datetime.datetime.now(), round_to_nearest=True ):
    '''Collect all dates between 10-28-2017 and today, that are located on either of the Qclock hands at the specified clock time.
    <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the aligned dates.
             NB. For a digital time of 04:20 the hour-hand on the analog clock is pointing at precisely 21.67 minutes, which would be
             returned here as the 22nd minute if <round_to_nearest>=True, else as the 21st minute;
    Returns a tuple with 2 lists containing datetime.datetime objects: the first list contains the dates located on the hour-hand, and the
    second list contains the dates located on the minute-hand of the Qclock.'''
    # import datetime
    hour_hand, minute_hand, roundoff = [], [], [int,round][bool(round_to_nearest)]
    d_today = datetime.datetime.today().toordinal()
    d_start = 736630   # QClock start date: '10-28-2017'
    if isinstance( qtime, ( datetime.datetime, datetime.time ) ): H, M = qtime.hour, qtime.minute
    elif isinstance( qtime, (tuple,list) ) and len( qtime ) > 1:  H, M = qtime[0], qtime[1]
    else: return [], []
    try: H, M = int( H ) % 12, int( M ) % 60
    except: return [], []
    H_date = d_start + roundoff( ( ( H + 8 ) % 12 + M / 60 ) * 5 ) # rounded to floor or nearest.
    M_date = d_start + ( M + 40 ) % 60
    while H_date <= d_today:
        hour_hand.append( datetime.datetime.fromordinal( H_date ) ); H_date += 60
    while M_date <= d_today:
        minute_hand.append( datetime.datetime.fromordinal( M_date ) ); M_date += 60
    return hour_hand, minute_hand


def qclock_get_aligned_qposts_for_date( qdate=datetime.datetime.today(), include_mirror=True, print_list=True ):
    '''Collects all earlier Qposts posted on a date that aligns with the given <qdate> on the Qclock.
    <qdate>: a datetime.datetime object or a string representing the date for which to retrieve all earlier aligned Qposts.
    <include_mirror>: Boolean determining whether to also include the aligned Qposts from the opposite side of the Qclock center.
    <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
    Returns a single list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the input date.'''
    # import datetime
    dt, dts, aligned, mirror = qclock_get_aligned_dates_for_date( qdate, include_mirror )
    if dts:
        result, s_aligned, s_mirror = [], [], []
        for d in aligned:
            qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
            result.extend( qposts_for_day )
            if print_list: s_aligned.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
        if print_list: print( f'{tc}Qclock Directly aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_aligned )}' )
        if include_mirror:
            for d in mirror:
                qposts_for_day = get_qposts_for_date( d ); n_qp = len(qposts_for_day)
                result.extend( qposts_for_day )
                if print_list: s_mirror.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
            if print_list: print( f'{tc}Qclock Opposite aligned dates for {bw} {dts} {ts+tc}:{ts}\n{", ".join( s_mirror )}' )
        return result


def qclock_get_aligned_qposts_for_clocktime( qtime, round_to_nearest=True, print_list=True ):
    '''Returns a list of tuples(QmapID, Qpost) for all Qposts whose date is located on one of the hands of the Qclock on the given clocktime.
    <qtime>: A datetime.datetime, datetime.time, or a 2-tuple(H,M) specifying the digital time for which to retrieve the Qclock-aligned Qposts.
    <print_list>: If True, it prints out the dates on the Qclock-hands, and the number of Qposts that were posted on each of those dates.'''
    result, s_hour, s_minute = [], [], []
    hour_hand, minute_hand = qclock_get_aligned_dates_for_clocktime( qtime, round_to_nearest )
    for d in hour_hand:
        qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
        result.extend( qposts_for_day )
        if print_list: s_hour.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
    for d in minute_hand:
        qposts_for_day = get_qposts_for_date( d ); n_qp = len( qposts_for_day )
        if hour_hand != minute_hand: result.extend( qposts_for_day )
        if print_list: s_minute.append( f'{d.strftime(_DT_FORMAT)} ({bl if n_qp > 0 else bw} {n_qp} {ts})' )
    if print_list:
        print( f'{tc}Qclock Hour-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_hour )}' )
        print( f'{tc}Qclock Minute-hand dates for clocktime ({qtime[0]:02d}:{qtime[1]:02d}):{ts}\n{", ".join( s_minute )}' )
    return result


def qclock_get_aligned_qposts_for_qmap_id( qmap_id, include_mirror=True, print_list=True ):
    '''Collects all earlier Qposts aligning on the Qclock with the Qpost of the given <qmap_id>.
    <qmap_id>: Integer Qmap ID number of the Qpost for which to get all Qclock-aligned Qposts.
    <include_mirror>: Boolean determining whether to also include the aligned dates from the opposite side of the Qclock center.
    <print_list>: If True, it prints out the aligned dates and the number of Qposts that were posted on each of those dates.
    Returns a list of tuples(QmapID, Qpost) for all Qposts whose date aligns on the Qclock with the post date of the given <qmap_id>.'''
    return qclock_get_aligned_qposts_for_date( get_qpost_dates_for_qmap_ids( [ qmap_id ] )[0], include_mirror, print_list )


def qposts_to_sqlite( qposts, url_save ):
    '''Exports a list of Qpost-dictionaries into an SQLite3 database; Only non-existing (new) records are added to the database.'''
    # import os, sqlite3
    try:
        qposts_reversed = qposts.copy()
        qposts_reversed.reverse()        # Reverse order of Qposts, so that the oldest Qpost gets qmap_id=1.
        connection = sqlite3.connect( url_save )  # Creates an empty db if the specified name is not found.
        cursor  = connection.cursor()
        success = 0
        str_sql  = "CREATE TABLE IF NOT EXISTS qposts (qmap_id INTEGER PRIMARY KEY, timestamp INTEGER, text TEXT, media TEXT, "
        str_sql += "refs TEXT, name TEXT, trip TEXT, userId TEXT, link TEXT, source TEXT, threadId TEXT, id TEXT, title TEXT, "
        str_sql += "subject TEXT, email TEXT, timestampDeletion INTEGER, CONSTRAINT unique_id_ts UNIQUE (id,timestamp));"
        cursor.execute( str_sql )        # Create table "qposts" if not already present.
        connection.commit()
        for qp in qposts_reversed:       # Add only the *new* qposts to the table.
            values = ( None, int( qp.get( 'timestamp', -1 ) ), qp.get( 'text', '' ), str( qp.get( 'media', '' ) ),
                       str( qp.get( 'references', '' ) ), qp.get( 'name', '' ), qp.get( 'trip', '' ), qp.get( 'userId', '' ),
                       qp.get( 'link', '' ),  qp.get( 'source', '' ),  qp.get( 'threadId', '' ), qp.get( 'id', '' ),
                       qp.get( 'title', '' ), qp.get( 'subject', '' ), qp.get( 'email', '' ), qp.get( 'timestampDeletion', None ) )
            try: cursor.execute( "INSERT INTO qposts VALUES( ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? );", values ); success += 1
            except: pass
        connection.commit()
        connection.close()
        filename = os.path.split( url_save )[1]
        hlink = ansi_hyperlink( f'file://{url_save}', filename, fg=(150,190,10), bg=(), style=(0,1,1,0,0) )
        print( f'{tc}Exported {bw} {success} {ts+tc} new Qpost records to SQLite3 Database "{hlink}{tc}".{ts}' )
    except Exception as e:
        safe_url = collapse_user( url_save )   # Hide user name before printing url.
        print( f'{er}Error while exporting Qpost records to SQLite3 Database "{safe_url}":{ts} {e}' )


def qposts_sqlite_query( url_db, sql='SELECT * FROM qposts;' ):
    '''Executes a SELECT query <sql> in the SQLite3 database <url_db>, and returns the result.'''
    # import os, sqlite3
    result = '<Error>'
    if os.path.exists( url_db ):
        try:
            connection = sqlite3.connect( url_db )
            cursor     = connection.cursor()
            result     = list( cursor.execute( sql ) )
            connection.close()
        except:
            safe_url = collapse_user( url_db )  # Hide user name before printing url.
            print( f'{er}SQL: Failed to execute query "{sql}" in SQLite3 Database "{safe_url}".{ts}' )
    return result


def qposts_sqlite_count_records( url_db, table='qposts' ):
    '''Returns the number of records in the table <table> of the SQLite3 database <url_db>.'''
    rec_count = 0
    try:
        connection = sqlite3.connect( url_db )
        cursor     = connection.cursor()
        cursor_ids = cursor.execute( f"SELECT * FROM {table};" )
        rec_count  = len( list( cursor_ids ) )
        connection.close()
    except: print( f'{er}Failed to count records in SQLite3 Database table "{table}".{ts}' )
    return rec_count


def qpost_text_cleanup( qp_text ):
    '''Cleanup leftover Html-formatting from the Qpost text field <qp_text>.'''
    # import re, html
    if qp_text:
        if qp_text[-1] == '\n': qp_text = qp_text[:-1]     # remove single newline character at the end.
        qp_text = re.sub( r'<strong>(.*?)</strong>', r'\1', qp_text )  # strong-tags --> bold.
        qp_text = re.sub( r'<em>(.*?)</em>', r'\1', qp_text )         # em-tags --> italic.
        qp_text = re.sub( r'<u>(.*?)</u>', r'\1', qp_text )           # u-tags --> underline.
        qp_text = re.sub( r'<span class="heading">(.*?)</span>', r'\1', qp_text )  # headings --> red bold.
        qp_text = re.sub( r'<span class="spoiler">(.*?)</span>', r'\1', qp_text )   # spoilers --> dimmed text.
        qp_text = re.sub( r'<span class="detected">(.*?)</span>', r'\1', qp_text )  # detected --> inverted color.
        qp_text = re.sub( r'<p class="body-line empty ">(.*?)</p>', r'\1', qp_text )     #body-line empty --> plain text.

        qp_text = re.sub( r'</p><p class="body-line ltr quote">', '', qp_text )  # remove faulty tag.
        qp_text = re.sub( r'</p><p class="body-line ltr ">', '', qp_text )       # remove faulty tag.
        return html.unescape( qp_text )            # Remove HTML escape-codes ( requires Python 3.4 )


def print_qpost( qp, lv='' ):
    '''Print elements of the specified Qpost.
    #   <qp> : dictionary representing a Qpost from Qanon.pub.
    #   <lv> : string representing the submenu depth level: pass _LVL_INDENT per added sublevel.'''

    # TIMESTAMP
    qp_timestamp = qp.get( 'timestamp', '' )
    qp_date      = datetime.datetime.fromtimestamp( qp_timestamp )
    qp_datestr   = qp_date.strftime( _DTM_FORMAT )
    qp_datediff  = datetime.datetime.now() - qp_date
    print( f'{lv}{tc}Timestamp: {ts}{qp_timestamp}\t{tc}Date: {ts}{qp_datestr}' )
    print( f'{lv}{tc}Ago: {ts}{qp_datediff}' )
    qp_ts_delete = qp.get( 'timestampDeletion', None )
    if qp_ts_delete:
        qp_ts_datestr = datetime.datetime.fromtimestamp( qp_ts_delete ).strftime( _DTM_FORMAT )
        print( f'{lv}{tc}Timestamp Deletion: {ts}📍{qp_ts_delete}\t{tc}Date: {ts}{qp_ts_datestr}' )

    # NAME
    qp_name   = qp.get( 'name', '<None>' )
    qp_trip   = qp.get( 'trip', '<None>' )
    qp_userId = qp.get( 'userId', '<None>' )
    print( f'{lv}{tc}Name: {ts}{qp_name}\t\t\t{tc}Tripcode: {ts}{qp_trip}\t{tc}User ID: {ts}{qp_userId}' )

    # SOURCE
    qp_source   = qp.get( 'source', '' )
    qp_threadId = qp.get( 'threadId', '' )   # this tag can be missing in some posts.
    qp_id       = qp.get( 'id', '' )
    qp_link     = qp.get( 'link', '' )
    print( f'{lv}{tc}Source: {ts}{qp_source}\t{tc}Thread ID: {ts}{qp_threadId}\t{tc}Post ID: {ts}{qp_id}' )
    print( f'{lv}{tc}Source Link: {ts}{qp_link}' )

    #TITLE/SUBJECT
    for title in [ 'subject', 'title' ]:
        qp_title = qp.get( title, None )
        if qp_title is not None: print( f'{lv}{tc}{title.title()}: {ts}{qp_title}' )

    # TEXT
    qp_text = qp.get( 'text', '' )
    print( f'{lv}{tc}Text: {ts}' )
    if qp_text:
        qp_text = qpost_text_cleanup( qp_text )

        if _MAX_WRAP > 0:
            lines   = []
            for txline in qp_text.splitlines():   # wrap text to a fixed width.
                lines.extend( wrap( txline, _MAX_WRAP, break_long_words=False, break_on_hyphens=False, initial_indent=lv, subsequent_indent=lv ) )
            qp_text = '\n'.join( lines )

        if _SHOW_ABBR_TOOLTIPS:
            for abbr in _QPOST_ABBR:          # add Tooltip to known abbreviations.
                if abbr in qp_text:
                    qp_text = re.sub( r"(\W|\n|^)" + re.escape(abbr) + r"(\W|\n|$)", r"\1" +
                    ansi_hyperlink( _QPOST_ABBR[abbr], abbr ) + r"\2", qp_text )

        print( f'{qp_text}' )

    # MEDIA
    qp_media = qp.get( 'media', [] )
    if qp_media is None: qp_media = []
    print( f'{lv}{tc}Media: {ts}{len(qp_media)}' )
    for q_pic in qp_media:
        print( f"{lv}{_LVL_INDENT} {q_pic['url']}" )
        print( f"{lv}{_LVL_INDENT} {q_pic['filename']}" )

    # REFERENCES
    qp_refs = qp.get( 'references', [] )     # this tag is only present if the Qpost has references.
    print( f'{lv}{tc}References: {ts}{len(qp_refs)}' )
    for q_ref in qp_refs:
        print_qpost( q_ref, lv + _LVL_INDENT )


def find_regexp_groups( tuples, regexp ):
    '''Returns a list of matched groups from <regexp>.
    <tuples>: List of tuples(QmapID,str_or_dict).
    <regexp>: The regular expression to match in the 2nd element of <tuples>; parenthesized groups are returned.
              This argument should be passed as a raw string, e.g. fr"{regexp}".'''
    #import re
    ret = []
    for item in tuples:
        s = re.findall( regexp, str(item[1]) )
        if s and len( s ) > 0: ret.append( (item[0], s) )
    return ret


def print_qpost_tuples( qp_tuples, key='' ):
    '''Display results returned by search_qposts(), search_qposts_by_date(), get_qposts_by_id().'''
    for qp in qp_tuples:
        print( f'\n{tc}Qmap ID:{ts} {qp[0]}' )
        if isinstance( qp[1], dict ): print_qpost( qp[1] )
        else: print( f"{tc}{key}:{ts} {qp[1] if key != 'text' else qpost_text_cleanup(qp[1]) }" )


def print_unique_qpost_field_values( qp_tuples, key='trip' ):
    '''Display unique values found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
    qp_unique = set( [ qp[1] for qp in qp_tuples ] )
    print( f'{tc}Unique values for key=\'{key}\':{ts} {qp_unique}' )


def print_qpost_field_frequency_list( qp_tuples, key='', lex=0 ):
    '''Display a list of character/word frequencies found in the Qpost field <key> of the Qposts in <qp_tuples>.'''
    qp_freqlist = [ ( qp[0], count_frequencies( qp[1], lex=lex ) ) for qp in qp_tuples ]
    print( f"{tc}{('Character','Word')[min(max(0,lex),1)]} Frequency list for key=\'{key}\':{ts} {qp_freqlist}" )


def qposts_terminal_loop():
    '''Start an input loop in the Terminal where the user can perform various operations on the _QPOSTS list.'''
    global _QMAP_IDS, _QPOSTS
    n_posts = len(_QPOSTS)
    choice  = 0
    duration_options = [ (f'     {tc}1{ts}:  as a number of seconds.', ['1'], ''),
                         (f'     {tc}2{ts}:  in the format (H)HH:MM:SS.', ['2'], ''),
                         (f'     {tc}3{ts}:  in the format DAYS, HH:MM:SS.', ['3'], ''),
                         (f'     {tc}4{ts}:  in the format 1w2d3h4m5s.', ['4'], ''),
                         (f'     {tc}5{ts}:  as a verbose duration string.', ['5'], '') ]
    sb_overwrite = f'{cu+gr}Search results will overwrite the current subset.{ts}'
    options = [ (f' {tc+bd}A)  {ts+tc}CURRENT SUBSET{ts}', 'A'),
                (f' {tc}1{ts}:  Define a new subset of Qmap IDs.', ['1'], 'A'),
                (f' {tc}2{ts}:  Display the current subset of {bw}' + ' {} ' + f'{ts} Qmap IDs.', ['2'], 'A'),
                (f' {tc}3{ts}:  Display (a field of) all Qposts from the current subset.', ['3'], 'A'),
                (f' {tc}4{ts}:  Display unique values from a field of all Qposts from the current subset.', ['4'], 'A'),
                (f' {tc}5{ts}:  Display a list of formatted datetimes of all Qposts from the current subset.', ['5'], 'A'),
                (f' {tc}6{ts}:  Display a list of relative time-intervals between the Qposts from the current subset.', ['6'], 'A'),
                (f' {tc}7{ts}:  Display a list of character/word-frequencies for Qposts from the current subset.', ['7'], 'A'),
                (f' {tc}8{ts}:  Find matching Regular Expression groups in the Qposts from the current subset.', ['8'], 'A'),
                (f' {tc}9{ts}:  Find the longest common substring in (a field of) all Qposts from the current subset.', ['9'], 'A'),
                (f' {tc+bd}B)  {ts+tc}SEARCH ALL QPOSTS{ts}', 'B'),
                (f'{tc}10{ts}:  Case-sensitive text search in (a field of) all {bw}' + ' {} ' + f'{ts} Qposts; {sb_overwrite}', ['10'], 'B'),
                (f'{tc}11{ts}:  Find all Qposts matching a Regular Expression; {sb_overwrite}', ['11'], 'B'),
                (f'{tc}12{ts}:  Date/Time search in all Qposts; {sb_overwrite}', ['12'], 'B'),
                (f'{tc}13{ts}:  Find all (earlier) Q-Clock aligned Qposts for a given date; {sb_overwrite}', ['13'], 'B'),
                (f'{tc}14{ts}:  Find all (earlier) Q-Clock aligned Qposts for a given Qmap ID; {sb_overwrite}', ['14'], 'B'),
                (f'{tc}15{ts}:  Find all Q-Clock aligned Qposts for a given clock time (HH:MM); {sb_overwrite}', ['15'], 'B'),
                (f' {tc+bd}C)  {ts+tc}DOWNLOAD / SAVE{ts}', 'C'),
                (f'{tc}16{ts}:  Check online if there are new Qposts available.', ['16'], 'C'),
                (f'{tc}17{ts}:  Download the latest Qposts JSON-file from qanon.pub to your Downloads folder.', ['17'], 'C'),
                (f'{tc}18{ts}:  Export all (new) Qposts to an SQLite3 Database inside your Downloads folder.', ['18'], 'C'),
                (f'{tc}19{ts}:  Download images from the media/image URLs from all Qposts in the current subset.', ['19'], 'C') ]
    db_options = [ (f' {tc+bd}D)  {ts+tc}SQLITE3 QPOSTS DATABASE{ts}', 'D'),
                (f'{tc}20{ts}:  Display the results of an SQL SELECT-query from your Qposts Database.', ['20'], 'D') ]
    message = [ f'\n{tc+iv+cu}Qposts research tools menu:{ts}',
                f'{mc}Please choose one of the menu options ( or q to quit ):{ts} ',
                f'{tc}Type one of the commands: ' +
                f_all( ['all','first N','last N','prev N','next N','reverse','sort up','sort down'], f'{mc+iv}', f'{ts}', ' ' ) +
                f"{tc},{ts}\n{tc}or type a series of Qmap IDs and/or Qmap ID subranges separated by comma's,{ts}\n" +
                f"{tc}where Qmap ID subranges can be specified by placing a dash '{mc+iv}-{ts+tc}' in between two Qmap IDs,{ts}\n" +
                f"{tc}and where an asterisk '{mc+iv}*{ts+tc}' represents the most recent Qmap ID, for example:{ts+gr} 5-8,2777-*{ts}\n" +
                f"{mc}Please enter a new subset of Qmap IDs:{ts} ",
                f"{er}Incorrect list format: should only be integers separated by comma's, for example:{ts} 1,2,82,14",
                f'{er}Incorrect Qmap ID: the numbers should be from 1 to {n_posts}.{ts}',
                f'{er}Incorrect range format: should be 2 integers separated by a hyphen, for example:{ts} 1-10',
                f'{tc+cu}Fields:{ts} ' + f_all( _QPOST_KEYS, f'{mc+iv}', f'{ts}', ' ' ) +
                f'{ts}\n{mc}Please enter a field name/number (or nothing for the whole record):{ts} ',
                f'{tc+cu}Current subset of Qmap IDs:{ts}',
                f'{tc}Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
                f' and using {mc+iv}(parentheses){tc}.{ts}\n{tc}Atoms containing spaces, keywords or parentheses ' +
                f'should be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
                f'{mc}Please enter a Case-sensitive search term:{ts} ',
                f'{tc}Date search terms must have the format: "OP DATETIME" (including quotation marks),{ts}\n' +
                f'{tc}where OP is one of the comparison operators ' + f_all( ['on','>=','<=','!=','=','>','<'], f'{mc+iv}', f'{tc}' ) +
                f',{ts}\n{tc}and where DATETIME is either a timestamp or a verbose date string like "28 Oct 2017".{ts}\n' +
                f'{tc}Both parts "OP DATETIME" together must be enclosed in {mc+iv}"double quotation marks"{tc}.{ts}\n' +
                f'{tc}Date Search terms can be combined using the keywords ' + f_all( ['and','or','not'], f'{mc+iv}', f'{tc}') +
                f', and using {mc+iv}(parentheses){tc}.{ts}\n{tc}The {mc+iv}on{tc} operator selects all posts from the same day, ' +
                f'for example:{ts} \"on 11 nov 2019\"\n{mc}Please enter a date search term:{ts} ',
                f'{tc+cu}Relative durations between Qposts:{ts}',
                f'{tc+cu}Formatted posting datetimes:{ts}',
                f'{tc}Interval Durations can be represented in one of the following formats:{ts}',
                f'{mc}Please enter a date after 28 October 2017:{ts} ',
                f'{mc}Please enter a valid Qmap ID number:{ts} ',
                f'{cu+gr} 📎 See ' + ansi_hyperlink(_DOC_STRFTIME,"strftime format codes",style=(0,1,1,0,0)) + f':\n' +
                f' {mc+iv}%d{tc} = day number (01 to 31);         {mc+iv}%U{tc} = week number (00 to 53);    {ts}\n' +
                f' {mc+iv}%A{tc} = weekday;  {mc+iv}%a{tc} = weekday abbr.;  {mc+iv}%w{tc} = weekday number (0 to 6);   {ts}\n' +
                f' {mc+iv}%B{tc} = month;    {mc+iv}%b{tc} = month abbr.;    {mc+iv}%m{tc} = month number (01 to 12);   {ts}\n' +
                f' {mc+iv}%y{tc} = year number (00 to 99);        {mc+iv}%Y{tc} = year number (0000 to 9999);{ts}\n' +
                f' {mc+iv}%H{tc} = hour (00 to 23);    {mc+iv}%I{tc} = hour (01 to 12);    {mc+iv}%p{tc} = "AM" or "PM";{ts}\n' +
                f' {mc+iv}%M{tc} = minute (00 to 59);  {mc+iv}%S{tc} = second (00 to 59);  {mc+iv}%s{tc} = timestamp;   {ts}\n' +
                f' {mc+iv}%c{tc} = datetime;           {mc+iv}%x{tc} = date;               {mc+iv}%X{tc} = time.        {ts}\n' +
                f'{tc} {cu}The default format string is a long datetime: {mc+iv}{_DTM_FORMAT}{ts}\n' +
                f'{mc}Please enter a valid strftime format string (or nothing for default):{ts} ',
                f'{mc}Please enter a digital clocktime Hour and Minute (H:M):{ts} ',
                f'{tc}NB. at' + ' {} digital time, the Hour-hand on the analogue Qclock points to the {}-minute mark{},' +
                f'{ts}\n{tc}and forms a' + ' {}-degree angle with the Minute-hand.' + f'{ts}',
                f'{tc}The symbol {mc+iv}^{tc} matches the start of the text, {mc+iv}${tc} matches the end of the text, and {mc+iv}\\n{tc} ' +
                f'matches a newline character.{ts}\n{tc}Matched groups can be captured within parentheses, e.g.{ts} These people are (.*)\\n\n' +
                f'{mc}Please enter a Regular Expression ( capturing groups ):{ts} ',
                f'{mc}Please enter a Regular Expression pattern to find in all Qposts:{ts} ',
                f'{mc}Enter the lexical element to count: 0 = characters, or 1 = words:{ts} ',
                f'{mc}Also download images from referenced posts? [Y/n]:{ts} ',
                f'{tc} Database    :{ts} ' + ansi_hyperlink( f'file://{_URL_QPOSTS_DB}', _SAFE_URL_QPOSTS_DB, fg=(150,190,10) ) + '\n' +
                f'{tc} Table name  :{ts} {mc+iv}qposts{ts}\n' +
                f'{tc} Field names :{ts} ' + f_all( _QPOST_DB_COLS, f'{mc+iv}', f'{ts}', ' ') + f'\n' +
                f'{tc} Query Format:{ts} {mf}SELECT {cu}<fields>{mf} FROM {cu}qposts{mf} WHERE {cu}<condition>{mf};{ts}\n' +
                f"{tc} For Example :{ts} SELECT qmap_id, text FROM qposts WHERE strftime( '%w/%m', timestamp, 'unixepoch')='4/05';\n" +
                f"{' '*15}{cu+gr}( This example above finds all Qposts that are posted on a Friday in May ){ts}\n" +
                f"{' '*15}{cu+gr}📎  See " + ansi_hyperlink(_DOC_SQLITE_SLCT,'SQLite Select',style=(0,1,1,0,0)) +
                f'{cu+gr} (online) for more information about SQLite SELECT queries.{ts}\n' +
                f"{' '*15}{cu+gr}📎  See " + ansi_hyperlink(_DOC_SQLITE_FUNC,'SQL Functions',style=(0,1,1,0,0)) +
                f'{cu+gr} (online) for more functions that can be used inside the query.{ts}\n' +
                f'{mc}Please enter a valid SQL SELECT query:{ts} ',
                f'{er}Incorrect SELECT Query: must start with the word SELECT followed by a space and a value.{ts}' ]
    def input_qpost_key( default='' ):
        '''Ask user to enter a valid qpost field key; else it returns the given default.'''
        k, answer  = 0, input( message[6] )
        if answer.isdigit(): k = int( answer )
        return _QPOST_KEYS[k-1] if k > 0 and k <= len( _QPOST_KEYS ) else ( answer if answer in _QPOST_KEYS else default )
    def handle_search_results( tuples, key, print_tuples=True ):
        '''Display the search results specified by <tuples> and <key>; returns the new subset of Qmap IDs.'''
        qmap_ids = [ qpt[0] for qpt in tuples ] if tuples else []   # construct Current Subset.
        if print_tuples: print_qpost_tuples( tuples, key )
        print( message[7], integer_list_to_range_string( qmap_ids ) )
        return qmap_ids
    while choice not in ['_EXIT','_ERROR']:
        menu = options.copy()
        menu[2] = (options[2][0].format( len(_QMAP_IDS) ), options[2][1], options[2][2])   # insert current subset size.
        menu[11] = (options[11][0].format( len(_QPOSTS) ), options[11][1], options[11][2]) # insert current number of Qposts.
        if os.path.exists( _URL_QPOSTS_DB ): menu.extend( db_options )
        choice = input_menu( menu, message[0], message[1], visible=_VISIBLE_MENU_GROUPS )
        if choice in menu[1][1]:                         #(1) DEFINE A SUBSET OF QMAP IDS:
            range_string = input( message[2] )
            rs_lower = range_string.lower()
            if rs_lower in [ 'reverse', 'rev' ]: _QMAP_IDS.reverse()
            elif rs_lower in [ 'sort down', 'sort desc', 'desc' ]: _QMAP_IDS.sort( reverse=True )
            elif rs_lower in [ 'sort up', 'sort asc', 'sort', 'asc' ]: _QMAP_IDS.sort()
            else:
                range_string = description_to_range_string( range_string, len(_QPOSTS), _QMAP_IDS )
                _QMAP_IDS = range_string_to_integer_list( range_string, validate=is_valid_qmap_id, msgs=message[3:6] )
            print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
        elif choice in menu[2][1]:                       #(2) DISPLAY CURRENT SUBSET OF QMAP IDS:
            print( message[7], integer_list_to_range_string( _QMAP_IDS ) )
        elif choice in menu[3][1]:                       #(3) DISPLAY QPOSTS (or FIELDS) FROM THE CURRENT SUBSET:
            answer  = input_qpost_key()
            tuples  = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
            print_qpost_tuples( tuples, answer )
        elif choice in menu[4][1]:                       #(4) DISPLAY UNIQUE VALUES FROM A FIELD IN THE QPOSTS FROM THE CURRENT SUBSET:
            answer  = input_qpost_key( 'trip' )
            tuples  = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
            print_unique_qpost_field_values( tuples, answer )
        elif choice in menu[5][1]:                       #(5) DISPLAY FORMATTED POSTING DATETIMES OF QPOSTS FROM THE CURRENT SUBSET:
            strftime_format = input( message[15] )
            if strftime_format == '': strftime_format = _DTM_FORMAT
            dates = get_qpost_dates_for_qmap_ids( _QMAP_IDS )
            print( message[11], [ d.strftime( strftime_format ) for d in dates ] )
        elif choice in menu[6][1]:                       #(6) DISPLAY RELATIVE TIME INTERVALS BETWEEN QPOSTS FROM THE CURRENT SUBSET:
            tuples  = get_qposts_by_id( _QMAP_IDS, key='timestamp', qmap_ids=True )
            i_option = input_menu( duration_options, message[12], message[1] )
            previous, durations = 0, []
            for i,tstamp in enumerate( tuples ):
                if i == 0: previous = tstamp[1]; durations.append( ( tstamp[0], 0 ) )
                else:
                    difference = tstamp[1] - previous
                    if i_option == '1' : durations.append( ( tstamp[0], difference ) )
                    elif i_option in ['2','3'] : durations.append( ( tstamp[0], seconds_to_HMS( difference, days=(i_option == 3) ) ) )
                    elif i_option == '4' : durations.append( ( tstamp[0], seconds_to_timestring( difference, separator='' ) ) )
                    elif i_option == '5' : durations.append( ( tstamp[0], seconds_to_timestring( difference, [], True, ', ' ) ) )
                    previous = tstamp[1]
            print( message[10], durations )
        elif choice in menu[7][1]:                       #(7) DISPLAY CHAR/WORD FREQUENCIES IN THE CURRENT SUBSET:
            lex = input( message[20] )
            lex = 0 if lex not in ['0','1'] else int( lex )
            answer  = input_qpost_key( 'text' )
            tuples  = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
            print_qpost_field_frequency_list( tuples, key=answer, lex=lex )
        elif choice in menu[8][1]:                       #(8) DISPLAY MATCHING REGEXP GROUPS IN THE CURRENT SUBSET:
            regexp  = input( message[18] )
            answer  = input_qpost_key()
            tuples  = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
            ret = find_regexp_groups( tuples, fr'{regexp}' )
            print( f'{tc}Matched Groups for key=\'{answer}\':{ts} ', ret )
        elif choice in menu[9][1]:                       #(9) FIND LONGEST COMMON SUBSTRING IN THE CURRENT SUBSET:
            answer  = input_qpost_key()
            tuples  = get_qposts_by_id( _QMAP_IDS, key=answer, qmap_ids=True )
            str_lcs = longest_common_substring( [ str(qpt[1]) for qpt in tuples ] )
            print( f'{tc}Longest Common Substring for key=\'{answer}\':{ts} "{str_lcs}"' )
        elif choice in menu[11][1]:                      #(10) SEARCH SUBSTRING IN ALL QPOSTS, CREATING A NEW SUBSET:
            search_term = input( message[8] )
            answer  = input_qpost_key()
            tuples  = search_qposts( search_term, key=answer )
            print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching search term="{search_term}".{ts}' )
            _QMAP_IDS = handle_search_results( tuples, answer )
        elif choice in menu[12][1]:                      #(11) FIND QPOSTS MATCHING A REGEXP, CREATING A NEW SUBSET:
            search_term_regexp = input( message[19] )
            answer  = input_qpost_key()
            tuples  = search_qposts_regexp( search_term_regexp, key=answer )
            print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching regexp="{search_term_regexp}".{ts}' )
            _QMAP_IDS = handle_search_results( tuples, answer )
        elif choice in menu[13][1]:                      #(12) DATE SEARCH IN ALL QPOSTS, CREATING A NEW SUBSET:
            date_condition = input( message[9] )
            answer  = input_qpost_key()
            tuples  = search_qposts_by_date( date_condition )
            print( f'{tc}Found {bw} {len(tuples)} {tc} Qposts matching date condition="{date_condition}".{ts}' )
            _QMAP_IDS = handle_search_results( tuples, answer )
        elif choice in menu[14][1]:                      #(13) SEARCH Q-CLOCK ALIGNED QPOSTS FOR DATE, CREATING A NEW SUBSET:
            date_string  = input( message[13] )
            if not date_string: date_string = 'today'
            tuples = qclock_get_aligned_qposts_for_date( date_string )
            print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for date={date_string}.{ts}' )
            _QMAP_IDS = handle_search_results( tuples, '', False )
        elif choice in menu[15][1]:                      #(14) SEARCH Q-CLOCK ALIGNED QPOSTS FOR QMAP ID, CREATING A NEW SUBSET:
            qmap_id  = input( message[14] )
            if qmap_id.isdigit() and is_valid_qmap_id( int( qmap_id ) ): qmap_id = int( qmap_id )
            else: qmap_id = n_posts
            tuples = qclock_get_aligned_qposts_for_qmap_id( qmap_id )
            print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qmap ID={qmap_id}.{ts}' )
            _QMAP_IDS = handle_search_results( tuples, '', False )
        elif choice in menu[16][1]:                      #(15) SEARCH Q-CLOCK ALIGNED QPOSTS FOR CLOCKTIME, CREATING A NEW SUBSET:
            s_time  = input( message[16] )
            s_parts = s_time.split()
            if len( s_parts ) < 2: s_parts = s_time.split( ':' )
            if len( s_parts ) < 2: s_parts = s_time.split( ',' )
            try: qtime = ( int( s_parts[0] ), int( s_parts[1] ) )
            except: qtime = (4,20)
            qtime = ( ( qtime[0] + qtime[1] // 60 ) % 12, qtime[1] % 60 )  # normalize clocktime.
            tuples = qclock_get_aligned_qposts_for_clocktime( qtime, _QCLOCK_ROUND_NEAREST )
            hands_angle = clocktime_to_angle( qtime[0], qtime[1] )
            hr_pos = clocktime_hourhand_pos( qtime[0], qtime[1] )
            hr_pos_rounded = f'{[int,round][_QCLOCK_ROUND_NEAREST](hr_pos)}'
            hr_pos_info = f'' if hr_pos == float( hr_pos_rounded ) else f' (rounded off from {hr_pos:0.2f})'
            sqtime = f'{qtime[0]:02d}:{qtime[1]:02d}'
            print( message[17].format( sqtime, hr_pos_rounded, hr_pos_info, f'{hands_angle:0.2f}' ) )
            print( f'{tc}Found {bw} {len(tuples)} {tc} aligned Qposts for Qclocktime {bw} ({sqtime}) {tc}.{ts}' )
            _QMAP_IDS = handle_search_results( tuples, '', False )
        elif choice in menu[18][1]:                      #(16) CHECK FOR NEW QPOSTS:
            n_posts = check_update_local_qposts()
        elif choice in menu[19][1]:                      #(17) DOWNLOAD QPOSTS JSON-FILE FROM QANON.PUB:
            qposts = download_qposts_json( _URL_QPOSTS )
            if qposts:
                _QPOSTS = qposts; n_posts = len(_QPOSTS)
                print( f'{tc}File "Qposts.json" with {bw} {n_posts} {tc} Qposts successfully downloaded and saved to your Downloads folder.{ts}' )
        elif choice in menu[20][1]:                      #(18) EXPORT QPOSTS TO SQLITE3 DATABASE:
            qposts_to_sqlite( _QPOSTS, _URL_QPOSTS_DB )
        elif choice in menu[21][1]:                      #(19) DOWNLOAD IMAGES FROM THE CURRENT SUBSET:
            answer = input( message[21] )
            download_qpost_images( _QMAP_IDS, answer.lower() not in ['n','no'] )
        elif choice in menu[23][1]:                      #(20) PERFORM SQL SELECT QUERY IN QPOST.DB:
            sql = input( message[22] )
            if sql.lower().startswith( 'select ' ):
                if not sql.endswith( ';' ): sql += ';'
                res = qposts_sqlite_query( _URL_QPOSTS_DB, sql )
                print( f'{tc}SQL Query Result:{ts} ', res )
            else: print( message[23] )
        else: pass
        ##### END of qposts_terminal_loop()


#################
# Auxiliary Methods:

def input_menu( menu, header='', prompt='> ', invalid='Invalid choice.', quits=['q','quit','exit'], visible={} ):
    '''Ask user to input a choice from a menu.
    Menuitems can be displayed in groups, which can be individually collapsed or expanded by entering the group key.
    To collapse or expand all groups at once, the user can enter the builtin commands HIDE or SHOW respectively.
    NB. Menuitems that are currently hidden, are not valid choices; First the menuitem must be made visible before it can be chosen.
    The function returns the key of the chosen menuitem, or '_EXIT' if the user chose to quit.
    <menu>    : An ordered list of tuples, each with either 2 or 3 elements:
                For a group-header item, pass a 2-tuple( str, str ) where the first element is the header text to be displayed
                (including its key), and the second element is a key which identifies a group of menuitems.
                For a choosable menu item, pass a 3-tuple( str, list, str ) where the first element is the displayed text for this menuitem
                (including its preferred key), the 2nd element is a list of keys that the user can enter to select this particular choice,
                and the 3rd element is the key of the group that this menuitem belongs to ( i.e. the 2nd element of a group-header item ).
    <header>  : string to be displayed before the list of menu choices; Leave empty if you don't want a menu header to be displayed.
    <prompt>  : string to be displayed after the list of menu choices; This string should be asking for user input.
    <invalid> : string to be displayed when the user input is not recognized.
    <quits>   : list of lowercase string commands that will exit the input loop (when typed in any case).
    <visible> : dict of {group:int} defining the initial visibility for each group; Updated in-place when user changes settings.'''
    _COLLAPSE = ['hide', 'collapse', 'none']  # Commands to collapse all menu groups.
    _EXPAND   = ['show', 'expand', 'all']     # Commands to expand all menu groups.
    def print_menu():
        m_keys = []
        if header: print( header )
        for item in menu:   # print visible menuitems and construct a list of valid keys:
            if isinstance( item, tuple ) and len( item ) >= 2:
                if len( item ) == 2: print( item[0] )
                elif visible.get( item[2], 1 ): m_keys.extend( item[1] ); print( item[0] )
        return m_keys
    ch, menuitem_keys, group_keys = '', [], []
    for item in menu:        # Populate visibility dict and list of group keys:
        if isinstance( item, tuple ) and len( item ) == 2:
            if isinstance( item[1], str ):
                if item[1] not in visible: visible[ item[1] ] = 1
                group_keys.append( item[1] )
    menuitem_keys = print_menu()                 # Display the menu:
    while ch not in menuitem_keys:
        ch = input( prompt )
        if ch.lower() in quits: return '_EXIT'   # User chose to Quit.
        if ch.lower() in _COLLAPSE:              # Hide all menu groups.
            for gv in visible: visible[ gv ] = 0
            menuitem_keys = print_menu()
        elif ch.lower() in _EXPAND:              # Show all menu groups.
            for gv in visible: visible[ gv ] = 1
            menuitem_keys = print_menu()
        elif ch in group_keys:                   # Hide/Show a specific menu group.
            visible[ ch ] = 1 - visible[ ch ]
            menuitem_keys = print_menu()
        elif ch in menuitem_keys: return ch      # User chose a menu option.
        elif invalid: print( invalid )
    return '_ERROR'


def ansi_hyperlink( uri, text='Ctrl-Click Here', fg=None, bg=None, style=None ):
    '''Returns a string that can be displayed as a Ctrl-clickable hyperlink in the terminal.
    User can also right-click on the link to popup a contextmenu with options 'Open Hyperlink' and 'Copy Hyperlink Address'.
    Hovering the mouse over the hyperlink will popup a Tooltip showing the target uri.
    Hyperlink targets are opened using the system's default application for the target type.
    <uri> should be an urlencoded string containing only ascii characters 32 to 126, starting with an uri scheme identifier.
          Supported uri schemes a.o.:  "http://", "https://", "ftp://", "file://", "mailto:".
    <text>: String representing the Ctrl-clickable text to be displayed.
    <fg>: None, or a 3-tuple of integers representing the RGB-values of the Foreground Color for <text>.
    <bg>: None, or a 3-tuple of integers representing the RGB-values of the Background Color for <text>.
    <style>: None, or a 5-tuple of Booleans for displaying <text> in Bold, Italic, Underline, Strikethrough, Blink.
    NB. if you pass <fg>, <bg>, and/or <style>, then any existing formatting of the text before the link will not be continued
    after the link. In that case, pass non-destructive ansi-code via <text> itself.'''
    if not (fg or bg or style): return fr"]8;;{uri}\\{text}]8;;\\"
    scv = [('','1;'),('','3;'),('','4;'),('','9;'),('','6;')]
    sbg = f"48;2;{bg[0]};{bg[1]};{bg[2]}" if bg and len( bg ) >= 3 else ''
    sfg = f"{';' if sbg else ''}38;2;{fg[0]};{fg[1]};{fg[2]}" if fg and len( fg ) >= 3 else ''
    stl = ''.join( [scv[i][bool(s)] for i,s in enumerate( style )] ) if style and len( style ) == 5 else ''
    stl = stl if sbg or sfg else stl[:-1]
    rst = '' if stl or sbg or sfg else ''
    return f"]8;;{uri}\\[{stl}{sbg}{sfg}m{text}{rst}]8;;\\"


def collapse_user( str_path ):  # inverse of os.path.expanduser()
    '''Replaces the user directory in a path by a tilde ( to hide the user name ).'''
    return str_path.replace( os.path.expanduser('~'), '~' )


def download_binary_file( url, save_url ):
    '''Tries to download a file from the Internet, and save it into the specified location <save_url>.
    Does NOT check if the file type indicated in <save_url> is the same as the file type from <url>.
    This function returns True if the download succeeded, else it returns False.'''
    def printq( msg ): print( msg ); return False
    response, headers = None, {'User-Agent': 'Mozilla/5.0'}
    try: response = urlopen( Request( url, headers=headers) )  # from urllib.request import Request, urlopen
    except URLError as e:                                      # from urllib.error import URLError
        return False; printq( f"{er}URLError; cannot download file '{url}.'\tReason: {e.reason}.{ts}" )
    except: printq( f"{er}Error downloading file '{url}'.{ts}" )
    if not response: return False
    elif response.status != 200:
        printq( f"{er}Download Failure: Response has status code {response.status}.{ts}" )
    else:
        try:
            with open( save_url, 'wb' ) as f: f.write( response.read() )
            return True
        except: pass # ConnectionResetError
    return False


def parse_datetime_string( datetime_string='now', format_as='%c', parserinfo=None ):
    '''Parses a datetime string such as "Sept 17th, 1984 at 01:30 AM", and returns a 3-tuple containing the parsed datetime,
       a string expressing the parsed datetime in the specified format, and a rest tuple of unparsed tokens.
       If the parsing failed, this function returns (None,'','').
       <datetime_string>: String specifying a datetime to be parsed; The string can also specify a timestamp.
       <format_as>: determines the format of the datetime string to be returned; Default "%c" is locale date&time format.
                    if the format is '' or None, a floating point timestamp will be returned instead of a string.
       NB. Uses the module dateutil; results are not so good for verbose date strings inside a sentence.'''
    #import dateutil.parser as dateparser    #import datetime
    dts = datetime_string.strip() if isinstance( datetime_string, str ) else 'now'
    if dts.lower() in [ 'now', 'today' ]:  dt, rest = datetime.datetime.now(), ()
    elif is_numeral( dts ):  dt, rest = datetime.datetime.fromtimestamp( float( dts ), tz=None ), ()   # interpret input as timestamp.
    else:
        try: dt, rest = dateparser.parse( dts, parserinfo=parserinfo, default=None, dayfirst=True, yearfirst=False,
                                          ignoretz=True, fuzzy_with_tokens=True )
        except ValueError: return (None,'','')  # input could not be parsed. (Pass a custom parserinfo for the local user language?).
        except: return (None,'','')             # OverflowError?: parsed date exceeds the largest valid C integer.
    if dt: return dt, dt.strftime( format_as ) if format_as else dt.timestamp(), rest
    return (None,'','')


def seconds_to_timestring( seconds, units=['w','d','h','m','s'], add_s=False, separator=' ' ):
    '''Turns a number of seconds into a human-readable duration string, expressed in weeks, days, hours, minutes, and seconds.
    <seconds>: The total number of seconds in the duration; Can pass a float, but the decimal part is not used.
    <units>  : Symbols for each of the 5 durations (week, day, hour, minute, second), or <None> to use the verbose English words.
    <add_s>  : If True, the character 's' will be appended after the unit symbol, if the number for that unit is larger than 1.
    <separator>: String to put in between each of the number/unit pairs.
    Adapted from function elapsed_time(): http://snipplr.com/view/5713/python-elapsedtime-human-readable-time-span-given-total-seconds/'''
    assert( isinstance( seconds, (int,float) ) )
    if not units or len(units) < 5: units = [' week',' day',' hour',' minute',' second']  # NB. space before the units.
    if seconds == 0: return '%s%s' % ( '0', units[-1] + ( '', 's' )[add_s] )
    if seconds <  0: return '-' + seconds_to_timestring( -seconds, units, add_s, separator )
    duration, lengths = [], [ 604800, 86400, 3600, 60, 1 ]
    for unit, length in zip( units, lengths ):
        value = seconds // length
        if value >= 1:
            seconds %= length
            duration.append( '%s%s' % ( str(value), (unit, (unit, unit + 's')[value > 1])[add_s] ) )
        if seconds < 1: break
    return separator.join( duration )


def seconds_to_HMS( seconds, microseconds=False, days=True ):
    '''Converts <seconds> into a string of the format "HH:MM:SS" with optional microseconds and/or days.'''
    assert( isinstance( seconds, (int,float) ) and isinstance( microseconds, bool ) and isinstance( days, bool ) )
    if seconds == 0: return '00:00:00' + ('.000000' if microseconds else '')
    if seconds <  0: return '-' + seconds_to_HMS( -seconds )
    minutes, seconds = divmod( seconds, 60 )
    hours,   minutes = divmod( minutes, 60 )
    msecs = f'{seconds:09.6f}' if microseconds else f'{int(seconds):02d}'
    if not days: return f'{int(hours):02d}:{int(minutes):02d}:{msecs}'
    ds, hours = divmod( hours, 24 )
    d = f"{ds} day{['s',''][bool(ds==1)]}, " if ds else ''
    return f'{d}{int(hours):02d}:{int(minutes):02d}:{msecs}'


def timestamp_day_bounds( timestamp ):
    '''Returns a 2-tuple with timestamps for the start & end of the day in which the specified <timestamp> falls.
    NB. The starting bound is Inclusive, the ending bound is Exclusive ( being the start of the next day ).'''
    ts_ordinal = datetime.datetime.fromtimestamp( timestamp, tz=None ).toordinal()   # Serial Day Number.
    ts_start   = datetime.datetime.fromordinal( ts_ordinal ).timestamp()             # Timestamp Day-Start.
    return ts_start, ts_start + 86400


def clocktime_to_angle( hour, minute ):
    '''''Returns a float representing the angle between the hour and minute hands of an analog clock showing the specified time.'''
    H = int( hour ) % 12
    M = int( minute ) % 60
    return H * 30 - M * 6 + M / 2


def clocktime_hourhand_pos( hour, minute ):
    '''''Returns a float representing the position (0-59.916666) of the Hour-hand of an analog clock showing the specified time.'''
    H = int( hour ) % 12
    M = int( minute ) % 60
    return ( H + M / 60 ) * 5


def integer_list_to_range_string( integer_list, sep='-' ):
    '''Returns a string representation of the specified list of integers <integer_list>, where
    ranges of consecutive integers are compacted to only their start- and end, separated by <sep>.
    E.g. for input = [1,2,3,4,5,6,7,8,55] it returns the string "1-8,55".'''
    parts, previous, direction = [], None, 0
    for n in integer_list:
        if isinstance( n, int ) and previous is not None:
            if direction == 0:
                start = previous
                if n - previous == 1: direction = 1
                elif n - previous == -1: direction = -1
                else: parts.append( str( previous ) )
            elif n - previous != direction:
                    direction = 0
                    parts.append( str( start ) + sep + str( previous ) )
        previous = n
    parts.append( str( previous ) if direction == 0 else str( start ) + sep + str( previous ) )
    return ','.join( parts )


def range_string_to_integer_list( range_string, sep='-', validate=None, msgs=[] ):
    '''Returns a list of integers based on the specified <range_string>.
    <range_string>: Comma-separated string of: numbers and/or <sep>-separated number ranges.
    <sep>: Symbol separating the start and end of the number ranges inside <range_string>.
    <validate>: None, or Pass a validation function that should accept an integer and return a Boolean.
    <msgs>: List of 3 error messages in case the input is: Not a Number, Invalid Number, Invalid Range.'''
    def print_msg( i ):
        if msgs and len(msgs) > i: print( msgs[i] )
    intlist, items = [], range_string.split( ',' )
    for item in items:
        if sep in item:  # separator: defines a range.
            range_ext = item.split( sep )[0:2]
            if range_ext[0].isdigit() and range_ext[1].isdigit():
                nm1, nm2 = int( range_ext[0] ), int( range_ext[1] )
                if not callable( validate ) or ( validate( nm1 ) and validate( nm2 ) ):
                    subrange = list( range( nm1, nm2 - 1, -1 ) if nm2 < nm1 else range( nm1, nm2 + 1 ) )
                    intlist.extend( subrange )
                else: print_msg( 1 ); break
            else: print_msg( 2 ); break
        elif item.isdigit():
            nm = int( item )
            if not callable( validate ) or validate( nm ): intlist.append( nm )
            else: print_msg( 1 ); break
        else: print_msg( 0 ); break
    return intlist


def description_to_range_string( desc, n_max, cur=[] ):
    '''Interpret commands: "all", "first N", "last N", "previous N", "next N", and asterisk shortcut: "*",
    into a range_string that can be converted by range_string_to_integer_list(). '''
    lcase, commands = desc.lower(), [ 'first', 'last', 'next', 'prev', 'previous', ]
    if lcase == 'all': return f'1-{n_max}'
    if any( lcase.startswith( cmd ) for cmd in commands ):
        parts = desc.split()
        amount = 1 if len(parts) < 2 or not parts[1].isdigit() else int( parts[1] )
        amount = min( max( 1, amount ), n_max )
        if lcase.startswith( 'first' ):  return f'1-{amount}'
        if lcase.startswith( 'last' ): return f'{n_max}-{n_max-amount+1}'
        cmin, cmax = ( min( cur ), max( cur ) ) if cur else ( 1, n_max )
        if lcase.startswith( 'next' ): return f'{min( max( 1, cmax + 1 ), n_max )}-{min( max( 1, cmax + amount ), n_max )}'
        if lcase.startswith( 'prev' ): return f'{min( max( 1, cmin - 1 ), n_max )}-{min( max( 1, cmin - amount ), n_max )}'
    else: return desc.replace( '*', f'{n_max}' )  # asterisk * means the maximum value <n_max>.


def count_frequencies( text, lex=0 ):
    '''Returns a list of 2-tuples(str, int) containing the count of each lexical element in <text>.
    <lex>: Determines which lexical element to count: 0=count characters; 1=count words.'''
    # from collections import Counter
    if isinstance( text, ( list, tuple, dict ) ): text = str( text )
    if isinstance( text, str ):
        if lex == 0: return Counter( text ).most_common()
        elif lex == 1: return Counter( text.split() ).most_common()
    return []


def longest_common_substring( data ):
    ''' Finds the longest common substring from a list of strings.'''
    # From https://stackoverflow.com/questions/2892931/longest-common-substring-from-more-than-two-strings-python/2894073#2894073
    substr = ''
    if len( data ) == 1: return data[0]  # only 1 element: return itself as longest string.
    if len( data ) > 1:
        d, n = data[0], len( data[0] )
        if n > 0:
            for i in range( n ):
                for j in range( n - i + 1 ):
                    if j > len( substr ) and is_common_substring( d[i:i+j], data ):
                        substr = d[i:i+j]
    return substr

def is_common_substring( find, data ):
    '''Used by longest_common_substring().'''
    if len( data ) < 1 or len( find ) < 1: return False
    for i in range( len(data) ):
        if find not in data[i]: return False
    return True


def is_numeral( s ):
    '''Returns True if the input string <s> represents either an integer number (e.g. '2500'), a floating
       point number (e.g. '2500.0'), a number expressed in scientific notation (e.g. '2.5E3'), or 'NaN'.'''
    try: _ = float( s ); return True
    except : return False


def f_all( s_list, s_before='', s_after='', sep=', ' ):
    return sep.join( [ s_before + k + s_after for k in s_list ] ) if s_list else ''


def split_list( lst ):
    '''Returns two "flat" lists: the first containing all items from the first dimension of <lst>,
    the second containing all items from the second and further dimensions of <lst>.'''
    def flatten_list( lst, newlist=[] ):
        for item in lst:
            if isinstance( item, list ): flatten_list( item, newlist )
            else: newlist.append( item )
    dim_one, dim_rest = [], []
    for item in lst:
        if isinstance( item, list ): flatten_list( item, dim_rest )
        else: dim_one.append( item)
    return dim_one, dim_rest


#################
# Class Simple_Logic_Expression

class Simple_Logic_Expression:
    '''Represents a simple logical expression such as 'A and B or not C'.
    Only supports the logical connectives And, Or, Not, and parentheses;
    Atoms containing spaces or parentheses should be enclosed in "(double) quotation marks".'''

    def __init__( self, str_expression, parse_format=1, eval_func=[], eval_args=(), operators=[] ):
        '''<str_expression>: String containing the simple logic expression to be parsed, e.g: 'A and B or not C'.
           <operators>    : List of 5 symbols for [And, Or, Not, Left Paren, Right Paren], that can be used in <str_expression>.
           <parse_format> : Determines the format of the parsed output:
                            0=list (prefix)     e.g:   ['or', ['and', 'A', 'B'], ['not', 'C']];
                            1=string (infix)    e.g:   "(A and B) or (not C)".
           <eval_func>    : pass an evaluation function that returns a Boolean for each atom used in <str_expression>,
                         or pass a list of atoms that are used in <str_expression>, and whose value is <True>.
           <eval_args>    : optional tuple of arguments to pass on to the evaluation function <eval_func>.'''
        if not operators or len( operators ) < 5: operators = [ 'and', 'or', 'not', '(', ')' ]
        self.expression = str_expression
        self.eval_func  = eval_func
        self.eval_args  = eval_args
        self.format     = min(max( 0, parse_format ), 1)  # 0=list (prefix); 1=string (infix).
        self._OP_AND    = operators[0]      #'and'
        self._OP_OR     = operators[1]      #'or'
        self._OP_NOT    = operators[2]      #'not'
        self._PAR_L     = operators[3]      #'('
        self._PAR_R     = operators[4]      #')'

    def parse( self, parse_format=None ):
        '''Parse the current expression, optionally overriding the current parse format.
        based on: https://www.howtobuildsoftware.com/index.php/how-do/gbu/string-algorithm-parsing-algorithm-
        to-add-implied-parentheses-in-boolean-expression'''

        def stream_starts_with( stream, token ):
            return stream[0:len(token)] == list(token)

        def pop( stream, token ):
            if stream_starts_with( stream, token ):
                del stream[0:len(token)]
                return True
            return False

        def parse_primary( stream ):
            if pop( stream, '"' ): return parse_enclosure( stream, '"', '"' )   # parse double quote.
            while pop( stream, ' ' ): pass
            if pop( stream, self._PAR_L ):
                e = parse_or( stream )
                pop( stream, self._PAR_R )
                return e
            return parse_atom( stream )

        def parse_enclosure( stream, enc_left, enc_right ):
            r = [ '', enc_left][self.format]     # keep/restore enclosure symbols if format=1.
            while stream and not pop( stream, enc_right ):
                r += stream.pop(0)
            while pop( stream, ' ' ): pass
            return r + [ '', enc_right][self.format]

        def parse_binary( stream, operator, func ):
            while pop( stream, ' ' ): pass
            es = [func( stream )]
            while pop( stream, operator ): es.append( func( stream ) )
            if self.format == 0: return [operator, *es] if len(es) > 1 else es[0]
            else: return self._PAR_L + ' {} '.format(operator).join(es) + self._PAR_R if len(es) > 1 else es[0]

        def parse_unary( stream ):
            while pop( stream, ' ' ): pass
            if pop( stream, self._OP_NOT ):
                if self.format == 0: return [ self._OP_NOT, parse_unary( stream ) ]
                else: return f'({self._OP_NOT} {parse_unary( stream )})'
            return parse_primary( stream )

        def parse_or( stream ):
            while pop( stream, ' ' ): pass
            p = parse_binary( stream, self._OP_OR, parse_and )
            return p if p else parse_unary( stream )

        def parse_and( stream ):
            while pop( stream, ' ' ): pass
            p = parse_binary( stream, self._OP_AND, parse_unary )
            return p if p else parse_unary( stream )

        def parse_atom( stream ):
            atom = ''
            while stream and not pop( stream, ' ' ) and not stream_starts_with( stream, self._PAR_R ):
                atom += stream.pop(0)
            return atom

        if parse_format is not None: self.format = parse_format
        #if not isinstance( self.expression, (list,str) ): return self.expression
        output = parse_or( list( self.expression ) )
        return output if self.format == 0 else output[1:-1] # Removes the outermost pair of parentheses.


    def evaluate( self, eval_func=None, eval_args=None ):
        '''Evaluate the current expression, optionally overriding the current atom evaluation function (or list).'''

        def evaluate_atom( atom ):
            if isinstance( self.eval_func, list ): return atom in self.eval_func
            if callable( self.eval_func ): return self.eval_func( atom, *self.eval_args )

        def evaluate_op( op_list ):
            operator = op_list[0]
            truthval = evaluate_output( op_list[1] )
            if operator == self._OP_NOT: return not truthval
            if operator == self._OP_AND:
                for arg in op_list[2:]:  truthval = truthval and evaluate_output( arg )
                return truthval
            if operator == self._OP_OR:
                for arg in op_list[2:]:  truthval = truthval or evaluate_output( arg )
                return truthval
            return truthval

        def evaluate_output( output ):
            if isinstance( output, list ): return evaluate_op( output )
            if isinstance( output, str ):  return evaluate_atom( output )
            return output

        if eval_func is not None: self.eval_func = eval_func
        if eval_args is not None: self.eval_args = eval_args
        if not isinstance( self.expression, (list,str) ): return evaluate_atom( self.expression )
        output = self.parse( parse_format=0 )   # must be parse_format=0 (=Prefix list).
        return evaluate_output( output )


    def find_in( self, text ):
        '''Interprets the current expression as a complex search-string, and checks if it matches the target <text>.'''
        def eval_atom( atom ): return atom in text
        return self.evaluate( eval_atom )


    def match_value( self, value, eval_func ):
        '''<eval_func>: required evaluation function taking at least 2 arguments: the current atom and <value>.'''
        return self.evaluate( eval_func, (value,) )

# End of Simple_Logic_Expression


def evaluate_comparison( value1, op, value2 ):
    if op.lower() in ['on']:    # Returns True if value2 is a timestamp falling on the same calendar day as value1.
        ts_start, ts_end = timestamp_day_bounds( value2 )
        return value1 >= ts_start and value1 < ts_end
    elif op in ['>=']:      return value1 >= value2
    elif op in ['<=']:      return value1 <= value2
    elif op in ['!=']:      return value1 != value2
    elif op in ['==', '=']: return value1 == value2
    elif op in ['>']:       return value1 >  value2
    elif op in ['<']:       return value1 <  value2


def evaluate_datetime_condition( dt_condition, timestamp ):
    '''Returns True if the expression "<timestamp><dt_condition>" evaluates as True.
    <dt_condition>: Numerical Timestamp, or a String of the format "COMP DATETIME" (including double quotation marks),
       where COMP is one of the comparison operators in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ],
       and where DATETIME is a valid datetime expression, a timestamp, or 'now'.
       The 'on'-operator yields True iff the <timestamp> falls on the same day as the DATETIME specified in <dt_condition>.
    <timestamp>: Integer or Float to be compared against the DATETIME specified in <dt_condition>.'''
    if isinstance( dt_condition, (int,float) ):  # interpret dt_condition as timestamp.
        return evaluate_comparison( timestamp, 'on', dt_condition )
    for op in [ 'on', '>=', '<=', '!=', '==', '=', '>', '<' ]:
        if dt_condition.startswith( op ):
            dt_string = dt_condition[len(op):].strip()
            if is_numeral( dt_string ): dts = float( dt_string )       # interpret dt_string as timestamp.
            else: dt, dts, rest = parse_datetime_string( dt_string, '' ) # slow; instead pass a timestamp for dt_condition.
            if dts: return evaluate_comparison( timestamp, op, dts )
    return False


def evaluate_numeric_condition( num_condition_string, num ):
    '''Returns True if the expression "<num><num_condition_string>" evaluates as True.
    <num_condition_string>: String of the format "COMP NUMBER" (including double quotation marks),
       where COMP is one of the comparison operators in [ '>=', '<=', '!=', '==', '=', '>', '<' ],
       and NUMBER is any number string that can be casted into a float, e.g.: '1', '2.2e3', etc.
    <num>: Integer or Float to be compared against the NUMBER specified in <num_condition_string>.'''
    for op in [ '>=', '<=', '!=', '==', '=', '>', '<' ]:
        if num_condition_string.startswith( op ):
            num_string = num_condition_string[len(op):]
            try: return evaluate_comparison( num, op, float( num_string ) )
            except: pass  # Error if float() fails.
    return False


#################
# Main:

if __name__ == '__main__':

    # Read optional commandline parameters to override default settings:
    # Type "python3 qposts_research.py -h" to see a list of options.
    descr  = 'Python script to facilitate research/analysis of Qposts.'
    epilog = 'Thank you for using %(prog)s ... WWG1WGA!'
    h_tips = 'Add this flag to hide tooltips for abbreviations.'
    h_wrap = 'Wrap width (in characters) for the Qposts Text field.'
    h_indn = 'Amount of indentation for nested references.'
    t_cols = ['black','red','green','yellow','blue','magenta','cyan','white']
    h_tcol = 'Label Color: ' + '; '.join([ f'{i}=[3{i}m{c}' for i,c in enumerate(t_cols)]) + '.'
    h_menu = 'Initial visibility (0 or 1) for each menu group.'
    h_qset = 'Initial subset of Qmap IDs, e.g. "all" or "last 100".'
    h_date = 'Strftime format to display a short date string.'
    h_dtm  = 'Strftime format to display a long date&time string.'
    h_vers = 'Show program version and exit.'

    formatter = argparse.MetavarTypeHelpFormatter
    parser    = argparse.ArgumentParser( description=descr, epilog=epilog, formatter_class=formatter )
    parser.add_argument( '-nt', '--notips', help=h_tips, default=(not _SHOW_ABBR_TOOLTIPS), action='store_true' )
    parser.add_argument( '-w', '--wrap', type=int, help=h_wrap, default=_MAX_WRAP )
    parser.add_argument( '-i', '--indent', type=int, help=h_indn, default=len( _LVL_INDENT ) )
    parser.add_argument( '-c', '--color', type=int, help=h_tcol, default=argparse.SUPPRESS )
    parser.add_argument( '-d', dest='date', type=str, help=h_date, default=_DT_FORMAT )
    parser.add_argument( '-dtm', dest='datetime', type=str, help=h_dtm, default=_DTM_FORMAT )
    parser.add_argument( '-m', dest='menu', type=int, help=h_menu, default=argparse.SUPPRESS, nargs='*' )
    parser.add_argument( '-s', dest='subset', type=str, help=h_qset, default=[_INITIAL_SUBSET], nargs='+' )
    parser.add_argument( '--version', action='version', help=h_vers, version='%(prog)s version 0.0.1b (20200117)' )
    args = parser.parse_args()

    _INITIAL_SUBSET = ' '.join( args.subset )
    _SHOW_ABBR_TOOLTIPS = not args.notips
    _LVL_INDENT   = ' ' * args.indent
    _MAX_WRAP           = args.wrap
    _DT_FORMAT          = args.date
    _DTM_FORMAT         = args.datetime
    if hasattr( args, 'color' ):
        if args.color >= 0 and args.color <= 7:
            tc = f'[0;3{args.color}m'
    if hasattr( args, 'menu' ):
        mg_len  = len( _VISIBLE_MENU_GROUPS )
        mg_keys = list( _VISIBLE_MENU_GROUPS.keys() )
        for i, vis in enumerate( args.menu ):
            if i < mg_len: _VISIBLE_MENU_GROUPS[mg_keys[i]] = int( bool( vis ) )


    print( f'{tc}{datetime.datetime.now().strftime( _DTM_FORMAT )}' )

    _QPOSTS  = open_qposts_json( _URL_QPOSTS )   # Open local Qposts.json file.
    if not _QPOSTS:
        print( f"{er}Local file Qposts.json not found:{ts} Attempting to download Qposts.json from qanon.pub ..." )
        _QPOSTS = download_qposts_json( _URL_QPOSTS )

    if _QPOSTS:
        n_qposts = check_update_local_qposts()   # Check for Latest Qposts online.
        print( f'{tc}Local Qposts.json: containing {bw} {n_qposts} {ts+tc} Qposts.{ts}' )

        if os.path.exists( _URL_QPOSTS_DB ):     # Check if SQLite3 QPosts.db exists:
            n_recs = qposts_sqlite_count_records( _URL_QPOSTS_DB )
            print( f'{tc}Local Qposts.db:   containing {bw} {n_recs} {ts+tc} Qpost Records.{ts}' )

        range_string = description_to_range_string( _INITIAL_SUBSET, n_qposts )
        _QMAP_IDS = range_string_to_integer_list( range_string )       # Create subset of Qmap_ID's.
        print( f'{tc+cu}Current subset of Qmap IDs:{ts}', integer_list_to_range_string( _QMAP_IDS ) )

        qposts_terminal_loop()               # Start terminal input loop.
    else: print( f"{er}No Qposts.json available:{tc} Exiting Program ...{ts}" )

quit()
# End