make_test_repo (2)

#!/bin/bash
# https://gitlab.com/sc0ttj/Pkg/merge_requests/21#note_262770355
# /etc/apt/sources.list should have the following line (non-free & contrib are optional)
#deb http://ftp.de.debian.org/debian buster main non-free contrib
. /etc/DISTRO_SPECS
UPDATE_REPO=yes #The repo will be updated if it doesn't exist.
INSTALL_WEBSERVER=yes #On puppylinux if webserver="busybox_httpd" sinccy httpd is included in puppy's busybox.
START_WEBSERVER=yes
UPDATE_HOSTS_FILE=yes #Adds an item to the host file for the test repo
DOWNLOAD_PACKAGES=yes #This will be done anyway if UPDATE_REPO=yes
TEST_CMD=ppa2pup
#webserver=nginx
USE_CONFIG_FILE_FOR_WEB_SERVER=yes #
webserver="busybox_httpd" #httpd is part of busybox, so if httpd is specified we shouldn't install any web server.
# Try running these web servers before installing anything

web_server_fallbacks=( "nginx" "apache" "lighttpd" "httpd" )
#If for some reason there is no webserver installed, then try installing one of these.

web_server_install_fallbacks=( "${web_server_fallbacks[@]}" )



port=80 #If port isn't 80 then maybe we need to run pkg with proxychains.

config_file="" #Leave this black for the default settings
#config_file=/etc/httpd2.conf

repo_name_in=${DISTRO_BINARY_COMPAT}-main #Comment
repo_url_in=http://archive.ubuntu.com/ubuntu/ #TODO: look this up if it isn't specified

repo_url_out=http://archive.ubuntu.cherrypicked.com/repo/ubuntu/
repo_name_out=${repo_name_in//-main/-cherrypicked}

[ -z "$repo_url_out" ] && localhost

#Uncoment the next line to use a previously generated awk program
#AWK_PRG_Path_in=/usr/share/pkg/testing/filter_repo
#Uncoment the next line to generate a new awk program
AWK_PRG_Path_out=/usr/share/pkg/testing/filter_repo
#Or comment out both lines above to use an inline (i.e. string) based awk program

########### Chose a repo file to filter

distro_in=${DISTRO_BINARY_COMPAT}; distro_out="$distro_in"
distro_ver_in=${DISTRO_COMPAT_VERSION}; distro_ver_out="$distro_ver_in"
stream_in=main; stream_out=cherrypicked

########### Some Document Paths

#https://stackoverflow.com/questions/10674867/nginx-default-public-www-location
#www_root=/usr/share/nginx/html #This is the default web folder for nginx in ubuntu
www_root=/var/www/html #This was also suggested
#www_root=/var/www/ #This is the default web folder for apache.
REPO_DB_DOC_FILE_in=/var/packages/Packages-${distro_in}-${distro_ver_in}-${stream_in}
#REPO_DB_DOC_FILE_out=~/Packages-${distro_ver_out}-${distro_ver_out}-${stream_out}

TMPDIR=/tmp
DL_FNAME=ppa_Packages.gz
DL_FILE_EX=.gz

arch='i386'
case $(uname -m) in
  i*86) arch='i386'  ;;
  *64)  arch='amd64' ;;
esac

repo_root_rel_path=repo/${distro_out}
repo_root_path=${www_root}/${repo_root_rel_path}
doc_path=${repo_root_path}/dists/${distro_ver_out}/${stream_out}/binary-${arch} #/Packages.gz


mkdir -p $doc_path #This is where we will save the repo db doc file on our system.

download_url_in=$(echo "$repo_url_in" | sed -e 's#/$##g')/dists/${distro_ver_in}/${stream_in}/binary-${arch}/Packages.gz
#download_url_out=$(echo "$repo_url_in" | sed -e 's#/$##g')/dists/${distro_ver_in}/${stream_in}/binary-${arch}/Packages.gz


function is_running(){
  local repo_url=$1
  repo_urls=( "$repo_url" "localhost" "127.0.0.1" )
  for a_repo_url in "${repo_urls[@]}"; do
    if [ -z "$(wget --spider -S "$repo_url" 2>&1 | awk '/HTTP\// {print $2}')" ]; then
      if [ "$repo_url" != "$a_repo_url" ]; then
        echo "Webserver is running but hostname mapping is broken" >&2
        echo "Check hosts file, dns cache or dns server"  >&2
      fi
      echo "true"
    else
      echo "false"
    fi
  done
}
function set_webserver_options(){
  local webserver=$1
  case "$webserver" in
  *)
    options=()
    for option in -p -h -c; do
      case "$option" in
      -p) [ ! -z "$PORT" ] && options+=( "$_p" "$PORT" ); ;;
      -h) [ ! -z "$www_root" ] && options+=( "$_h" "$www_root" ); ;;
      -c)
        [ -z "$config_file" ] && config_file="$set_config_file_rtn"
        [ ! -z "$config_file" ] && options+=( "$_c" "$config_file" ); ;;
      esac
    done
    "$WEB_SERVER_CMD" ${options[@]}; ;;
  esac
}
function set_config_file(){
  local webserver="$1"
  if [ -z "$config_file" ] && [ "$USE_CONFIG_FILE_FOR_WEB_SERVER" = yes ] ; then
    case "$webserver" in
    httpd|busybox_httpd)
      set_config_file_rtn=/usr/share/pkg/testing/webserver/httpd/httpd.conf
      if [ ! -f "$set_config_file_rtn" ]; then
        mkdir -p "$(dirname "$set_config_file_rtn")"
        echo "directory auto index" > "$set_config_file_rtn"
      fi
    esac

  fi
}
function run_webserver(){
  local webserver=$1
  set_config_file $1
  set_webserver_options $1
  "$WEB_SERVER_CMD" ${options[@]}
  if [  ! "`is_running`" = true ]; then
    if [ "`is_running`" = true ]; then
      do_action_result=success
    else
      do_action_result=failed
    fi
  else
    echo "Webserver already running" >&2
  fi
}
function get_cmd(){
  local webserver="$1"

  #First Get the webserver command
  case "$webserver" in
  "busybox_httpd"|"busybox")
    if  [ $(busybox --list | grep -c httpd) -gt 0 ]; then
      WEB_SERVER_CMD="busybox_httpd"
    fi; ;;
  *)
     if [ ! -z "`which "$webserver"`" ]; then
       WEB_SERVER_CMD="$webserver"
     fi; ;;
  esac
  #TODO then get the otions for the web server
  case "$webserver" in
  *)
    _p="-p" #The port Number
    _c="-c" #the option for the configuation file
    _h="-h" #the option for the webserver root (AKA home directory"
  esac
}
function do_action(){
  local web_server="$1"; shift
  local cmd="$1"; shift
  case "$cmd" in
  get_cmd)
    get_cmd "$web_server"
    if [ -z "$WEB_SERVER_CMD" ]; then
      do_action_result=failed
    else
      do_action_result=success
    fi; ;;
  run|run-install)
    [ -z "$WEB_SERVER_CMD" ] && get_cmd "$web_server"
    if [ -z "$WEB_SERVER_CMD" ] && [ "$INSTALL_WEBSERVER" = yes ]; then
      if [ "$cmd" = "run-install" ] || [ "$2" = "install" ]; then
        pkg --get "$web_server"
        get_cmd "$web_server"
      fi
    fi
    [ -z "$WEB_SERVER_CMD" ] &&  run $@
      if [ "$(is_running)" = true ]; then
        do_action_result=success
      else
        run_webserver "$web_server" #set_webserver_options
        #do_action_result=failed
      fi
  esac
}
function do_actions(){
  for action in "$@"; do
    if [ "$1" = get_cmd ] || [ "$1" = run -a "$2" != "install" ]; then
      web_servers=( "$webserver" "${web_server_fallbacks[@]}" )
    else
      web_servers=( "$webserver" "${web_server_install_fallbacks[@]}" )
    fi
    for a_web_server in "${web_servers[@]}"; do
      case "$a_web_server" in
      "busybox_httpd"|httpd|busybox)
        do_action "httpd" "$@";
        ;;
      *)
        do_action "$a_web_server" "$a_web_server"; ;;
      esac
      if [ "$do_action_result" = success ]; then
        break 2
      fi
    done
  done
}

function echo_filter_line(){
    read a_pkg_name
    echo "pkg_filter[\""$a_pkg_name"\"]=\"true\""
}
[ ! -f "${doc_path}/Packages.gz" ] && UPDATE_REPO=yes
if [ "$UPDATE_REPO" = "yes" ]; then
  export PKG_KEEP_EPOCH_IN_DOC_REPO=true
  pkg --repo-update


#fi
  DL_FNAME_Path=${TMPDIR}/${DL_FNAME}
  rm "$DL_FNAME_Path"
  DL_DOC_FILE_PATH=${DL_FNAME_Path%"$DL_FILE_EX"}
  rm "$DL_DOC_FILE_PATH"

  wget --quiet "$download_url_in" -O "$DL_FNAME_Path" 1>/dev/null \
    || download_failed=true

  gunzip ${TMPDIR}/${DL_FNAME}





  AWK_PRG_1=\
'BEGIN {FS="|"; OFS="|"}
{ if ($1 ~ /^[^|]+:[^|]+$/  ){
    print $1 "|" $2 "|" $3 #We might want to use some of these other fields for a different application
 }}'

  awk_filter_str='function init_filter(){'

  filter_lines_path=/tmp/filter_lines
  rm "$filter_lines_path"
  while read pkg_record; do
    #PKG_NAME_ONLY="$(echo "$pkg_record" | cut -f2 -d'|')"
    #write_filter_line "$PKG_NAME_ONLY"
    echo "$pkg_record" | cut -f2 -d'|' | echo_filter_line
  done < <( cat $REPO_DB_DOC_FILE_in | awk "$AWK_PRG_1" ) \
  | sort -R | head -n 3 >> "$filter_lines_path" #| tr "\n" " " )
  #cat $REPO_DB_DOC_FILE_in | awk "$AWK_PRG"
  #read -p "Press enter to continue"
  awk_filter_str="$awk_filter_str
$(cat $filter_lines_path)"'
}
function filter_accept(s){ #Return true if we are to print the result
  if ( pkg_filter[s] == "true" ){
    return "true"
  } else {
    return "false"
  }
}'

  AWK_PRG="$awk_filter_str"'
function stripEpoch(s){
  sub(/^([0-9]*:)?/,"",s)
  return  s
}
BEGIN {init_filter()}
/^Package:/ { PKG=$0; sub(/^Package: /,"",PKG); FILTER_ACTION=filter_accept(PKG)}
#/^Version:/ {VER=$0; sub(/^Package: /,"",VER);
{if (FILTER_ACTION == "true"){
    print $0
  }
}'

  if [ -z "$AWK_PRG_Path_in" ] && [ -z "$AWK_PRG_Path_out" ]; then
    awk "$AWK_PRG" "$DL_DOC_FILE_PATH" > ${doc_path}/Packages.gz
  else
    if [ -z "$AWK_PRG_Path_in" ]; then
      echo "#!`which gawk` -f" > "$AWK_PRG_Path_out"
      mkdir -p dirname "$(dirname $AWK_PRG_Path_out)"
      echo "$AWK_PRG" >> "$AWK_PRG_Path_out"
      chmod +x "${AWK_PRG_Path_out}"
      AWK_PRG_Path_in="${AWK_PRG_Path_out}"
    fi
    awk -f "$AWK_PRG_Path_in" "$DL_DOC_FILE_PATH"  > ${doc_path}/Packages
  fi
  gzip "${doc_path}/Packages" > "${doc_path}/Packages.gz"
fi
if [ "$UPDATE_REPO" = "yes" ] || [ "$DOWNLOAD_PACKAGES" = yes ]; then
  AWK_PRG_3=\
'/^Filename:/ {
     system("wget --quiet \"$repo_url_in\" -O \"" RROOT "/" FPATH "\" 1>/dev/null")
     }'
   cat "${doc_path}/Packages" | awk -v "RROOT=\"$repo_root_path\"" \
    "$AWK_PRG_3"
  do_actions run install #Starts the web server

  if [ $(pkg --repo-list | grep -c "$repo_name_out") -eq 0 ]; then
    ( exec <<< "$repo_name_out"
      pkg add-repo "$repo_url_out" "$distro_ver_out" "$stream_out" )
  fi
fi
if [ "$UPDATE_HOSTS_FILE" = yes ]; then
  protocol="${repo_url_out%%//*}"
  protocol_stripped="${repo_url_out#*//}"
  repo_domain_out="${protocol_stripped%%/*}"
  repo_domain_out="$protocol//$repo_domain_out"
  #TODO: maybe add port 80 ping test here.
  if [ $(grep -c -F "$repo_domain_out" /etc/hosts) = 0 ]; then
    echo "127.0.0.1" "$repo_domain_out" >> /etc/hosts
    #http://murga-linux.com/puppy/viewtopic.php?p=1045805#1045805
    service dns-clean restart || service nscd restart || \
        systemctl restart nscd.service || nscd -l hosts
  fi
fi
if [ "$START_WEBSERVER" = "yes" ]; then
  do_actions run install

  #if [ -z "$(wget --spider -S "http://localhost" 2>&1 | awk '/HTTP\// {print $2}')" ]; then
  #  case "$webserver" in
  #  *)
  #    options=()
  #    for option in -p -h -c; do
  #      case "$option" in
  #      -p) [ ! -z "$PORT" ] && options+=( "$_p" "$PORT" ); ;;
  #      -h) [ ! -z "$www_root" ] && options+=( "$_h" "$www_root" ); ;;
  #      -c) [ ! -z "$config_file" ] && options+=( "$_c" "$config_file" ); ;;
  #      esac
  #    done
  #    "$WEB_SERVER_CMD" ${options[@]}; ;;
  #  esac
  #fi
fi
case "$TEST_CMD" in
ppa2pup) PKG_PPA2PUP_FN=ppa2pup pkg --repo-update; ;;
ppa2pup_gawk) pkg --repo-update; ;;
esac