Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- #!/bin/bash
- # Golos.io Articles To CSV Converter
- # This script works in 3 parts:
- # 1. Loads user's articles from GOLOS.io.
- # 2. Loads each articles statistics.
- # 3. Converts statistics into one CSV file
- # Save as like "articles.sh", do "chmod +x articles.sh", and then run script like this:
- # ./articles.sh sxiii
- # Replace sxiii with your GOLOS.io nickname
- # Author: @sxiii
- # Date: 17.02.2017; 05:00 AM
- tempfile="temp.articles.temp" # Temp file (for articles list)
- nu="http://node.golos.ws" # Node URL
- af="articles" # Article folder
- mkdir $af
- echo "Welcome! Let's look at articles list of user @$1 - loading articles."
- # Loading articles list from user $1
- x=$(curl -s --data '{"method": "get_discussions_by_author_before_date", "params": ["'$1'","","2016-06-06T00:00:00","100"], "id": 1 }' $nu)
- echo $x | jq -r 'recurse(.result.url[])?' | grep "url\":" | awk -F\" '{ print $4 }' | awk -F"/" '{ print $4 }' >> $tempfile
- echo "Finished, I found total of "$(cat $tempfile | wc -l)" articles of user "$1
- # Creating a template for CSV file
- t=0
- echo -e "\"Title\",\"Date\",\"Comments\",\"Votes\",\"Gold\",\"Tag1\",\"Tag2\",\"Tag3\",\"Tag4\",\"Tag5\",\"Images\",\"Text length\"" >> $af/allarticles.$1
- # Starting a loop over $tempfile
- while read p; do
- ((t++))
- echo "Saving article $t: $p"
- y=$(curl -s --data '{"method": "get_content", "params": ["'$1'","'$p'"], "id": 1 }' $nu)
- echo $y | jq -r '.result.title,.result.created,.result.children,.result.net_votes,.result.total_payout_value,[.result.json_metadata|fromjson.tags],[.result.json_metadata|fromjson.image|length],[.result.body|length]' > $af/$p.$1
- # Following code takes data from saved articles file and builds an index article file
- title=$(sed '1q;d' $af/$p.$1)
- time=$(sed '2q;d' $af/$p.$1)
- comments=$(sed '3q;d' $af/$p.$1)
- votes=$(sed '4q;d' $af/$p.$1)
- gold=$(sed '5q;d' $af/$p.$1)
- t1=$(sed '8q;d' $af/$p.$1)
- t2=$(sed '9q;d' $af/$p.$1)
- t3=$(sed '10q;d' $af/$p.$1)
- t4=$(sed '11q;d' $af/$p.$1)
- t5=$(sed '12q;d' $af/$p.$1)
- images=$(sed '16q;d' $af/$p.$1)
- symbols=$(sed '19q;d' $af/$p.$1)
- echo -e "\"$title\",\"$time\",\"$comments\",\"$votes\",\"$gold\",$t1$t2$t3$t4$t5,\"$images\",\"$symbols\"" >> $af/allarticles.$1
- done < $tempfile
- # Remove tempfile if you like:
- rm $tempfile
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement