#!/bin/bash

[ "$1" = "" ] && exit 0

export PYTHONIOENCODING=utf-8

#exec 2>/dev/null

mydir=`dirname "$0"`

if [ xx`echo "$1" | grep -E -o '^tagme '` != "xx" ]
then
	tagme=1
fi

IFS="
"

# For ubitux
# url=`echo "$*" | grep -o -E 'https?:\/\/\S+ ?'`
url=`echo "$*" | grep -o -P '(?:https?://|ftp://|news://|mailto:|file://|\bwww\.)[\w\-\@;\/?:&=%\$.+!*\x27,~#]*(\([\w\-\@;\/?:&=%\$.+!*\x27,~#]*\)|[éèêâàìîïûùòöô\w\-\@;\/?:&=%\$+*~])+'`

tag=`echo "$1" | grep -o -E '#.*#'`

if [ "$url" != "" -a "$tag" == "" ]
then
	clean_url=`echo "$url" | cut -d" " -f1`

	for unique_url in `echo "$clean_url"`
	do

	useragent="Mozilla/5.0 (X11; Linux x86_64; rv:67.0) Gecko/20100101 Firefox/67.0"

# Sat 28 Nov 2020 11:57:58 PM JST
# icecast/mpd block
		[ ! -z $(echo "$clean_url" | grep -E 'http*://.*\:8000') ] && exit 0
		[ ! -z $(echo "$clean_url" | grep -E 'http*://.*\:8001') ] && exit 0
		[ ! -z $(echo "$clean_url" | grep -E 'http*://.*\:8002') ] && exit 0

# Sun Sep 10 20:46:38 JST 2017
# They do some progressive loading shit
# Fri Jul 27 06:00:23 JST 2018
# also, if it's a channel/user link, we keep doing the standard download, but with a known non-js user-agent
		if [ ! -z $(echo "$clean_url" | grep -E 'https*://w*\.*youtube\.com') ] || [ ! -z $(echo "$clean_url" | grep -E 'https*://w*\.*youtu\.be') ]
		then
			if [ ! -z $(echo "$clean_url" | grep -E '/user/') ] || [ ! -z $(echo "$clean_url" | grep -E '/channel/') ]
			then
				useragent="Lynx"
			else
				"$mydir"/youtube-dl -U >/dev/null 2>/dev/null
				yt_title=$("$mydir"/youtube-dl -se "$clean_url" 2>/dev/null | head -n5)
				if [ ! -z "$yt_title" ]
				then
					echo "URL: $yt_title"
				fi
				exit 0
			fi
		fi

# Wed Apr 20 14:51:17 CEST 2016
# add fake referrer
# Wed Sep 14 22:21:31 CEST 2016
# Ok, let's use GET instead of HEAD
		refer="`echo $unique_url | cut -d/ -f1,2,3`"

		if [ "`wget --referer $refer --user-agent=$useragent --tries 1 --timeout=15 --server-response $unique_url  -O /dev/null 2>&1 | head -n 50 | grep -i 'Content-Type:.*html'`" == "" ]
		then
			exit 1
		fi

# Mon Apr  4 20:40:18 CEST 2016
# fuck it, let's downlad in bash too
		dlfile=`mktemp`
		cookfile=`mktemp`
		chmod 600 "$dlfile" "$cookfile"

# Thu 31 Dec 2020 08:23:50 AM JST
# Using googlebot user-agent to have a parsable tweet
		if [ ! -z $(echo "$clean_url" | grep -E 'https*://.*twitter\.com') ]
		then
			useragent="Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)"
		fi

# Fri 12 Jun 2020 07:34:41 PM JST
# 9gag recaptcha bypass, let's see how long this is going to work
		if [ ! -z $(echo "$clean_url" | grep -E 'https*://.*9gag\.com') ]
		then
			useragent="Lynx/2.8.9rel.1 libwww-FM/2.14 SSL-MM/1.4.1 GNUTLS/3.6.5"
		fi

		if [ ! -z $(echo "$clean_url" | grep -E 'https*://w*\.*reddit\.com') ]
		then
			curl -L 30 -H 'Accept-Language: en-us' -e "$refer" -A "$useragent" -c "$cookfile" --cookie "over18=1" "$unique_url" 2> /dev/null | head -c1M > "$dlfile"
		else
			curl -L 30 -H 'Accept-Language: en-us' -e "$refer" -A "$useragent" -c "$cookfile" "$unique_url" 2> /dev/null | head -c1M > "$dlfile"
		fi

		ftype=`file --mime-type "$dlfile" | cut -d: -f2 | sed 's/ //g'`
		if [ "$ftype" == "application/gzip" ]
		then
			exfile=`mktemp`
			chmod 600 "$exfile"
			cat "$dlfile" | gunzip 2>/dev/null > "$exfile"
			rm "$dlfile"
			dlfile="$exfile"
		fi

		title=$(python2 `dirname "$0"`/url.py "$dlfile" | tr '\n' ' ' | tr '\t' ' ' | head -c 1024)
		rm "$dlfile" "$cookfile"

		if [ "$title" != "" ]
		then
			if [ "$tagme" == "1" ]
			then
				title_clean=`echo "$title" | sed -r 's/[,.&@#$%^_-{}*()]//g'`
				echo -n "URL: $unique_url "
				line=`echo "$title_clean" | awk -F " " 'BEGIN { printf("#") } // { for(i=1;i<=NF;i++){ printf("%s",$i); if(i<NF) printf(", "); } } END { printf("#") }'`
				echo "$line"
				"$mydir"/links.sh "_void_" "$unique_url $line"
			else
				echo "URL: $title"
			fi
		fi
	done
fi