LCSC Scraper

1.16 KB

#!/usr/bin/fish

# this script uses the fish shell and should dump every LCSC Part
#pretty much untested...
set cats 308 312 316 319 320 328 348 365 380 385 395 423 440 450 460 470 487 493 500 513 515 570 575 582 601 905 938 953 986 10991 11032

#as far as I can tell, both are needed. Copy from (anonymous) lcsc visit
set X_CSRF_TOKEN "X-CSRF-Token: ..."
set Cookie 'Cookie: ...'

function dump
  set c $argv[1] #category
  set p $argv[2] #page
  curl 'https://lcsc.com/api/products/search' \
     -H $X_CSRF_TOKEN \
     -H $Cookie \
     --data-raw "current_page=$p&category=$c&in_stock=false&is_RoHS=false&show_icon=false&search_content=" \
     > $c-$p.json
end

for c in $cats;
dump $c 1
#set total (jq ".result.total" $c-1.json)
set total (sed -e "s/.*total\"://;s/,.*//" $c-1.json)
#set pages (math "ceil($total/25)")
set pages (sed -e "s/.*total_page\"://;s/,.*//" $c-1.json)

echo "$total in category $c, pages $pages"
sleep 2
for p in (seq 2 $pages)
  echo dumping cat $c, page $p of $pages
  dump $c $p
  sleep 2  # enough to not hit the rate limit. Maybe 1 is ok.
end