Add some imageboard domains #18038

pipboy96 · 2019-05-30T08:18:08Z

Imageboard list is taken from https://github.com/ccd0/imageboards.json and processed by a script written in Go.

Be aware that most websites are NSFW!

Alexa top 1M

The rest

The script

The code is extremely bad, please don't use this as an example of how to write idiomatic Go.

package main

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"sort"
	"strings"
)

type record struct {
	Url string
	Rank int
}

func (s record) String() string {
	if s.Rank == 0 {
		return fmt.Sprintf("- [ ] %s", s.Url)
	} else {
		return fmt.Sprintf("- [ ] %s (%d)", s.Url, s.Rank)
	}
}

type byRank []record

func (s byRank) Len() int {
	return len(s)
}
func (s byRank) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
func (s byRank) Less(i, j int) bool {
	if s[i].Rank == 0 {
		if s[j].Rank == 0 {
			return s[i].Url < s[j].Url
		}
		return false
	}
	if s[j].Rank == 0 {
		return true
	}
	return s[i].Rank < s[j].Rank
}

func readDomains() []string {
	var arr []string

	file, err := os.Open("top-1m.csv")

	if err != nil {
		log.Fatal(err)
	}

	reader := bufio.NewReader(file)

	for {
		line, _, err := reader.ReadLine()

		if err == io.EOF {
			break
		}

		if err != nil {
			log.Fatal(err)
		}

		lineStr := string(line)

		if idx := strings.Index(lineStr, ","); idx != -1 {
			lineStr = strings.TrimPrefix(lineStr, "www.")
			arr = append(arr, lineStr[idx + 1:])
		} else {
			log.Fatal(line)
		}
	}

	return arr
}

func main() {
	alexa := readDomains()

	file, err := ioutil.ReadFile("imageboards.json")

	if err != nil {
		log.Fatal(err)
	}

	var imageboards []struct{
		Url string `json:"url"`
	}

	err = json.Unmarshal(file, &imageboards)

	if err != nil {
		log.Fatal(err)
	}

	var domains []record

	for _, img := range imageboards {
		imgurl, err := url.Parse(img.Url)

		if err != nil {
			log.Fatal(err)
		}

		imghost := imgurl.Hostname()

		imghost = strings.TrimPrefix(imghost, "www.")

		rank := 0

		for n, domain := range alexa {
			if imghost == domain {
				rank = n
			}
		}

		domains = append(domains, record{imghost, rank})
	}

	sort.Sort(byRank(domains))

	for _, domain := range domains {
		fmt.Println(domain)
	}
}

pipboy96 added new-ruleset good first issue good volunteer task labels May 30, 2019

pipboy96 mentioned this issue Jun 20, 2019

Sites that should have https forced #3069

Open

121 of 524 tasks complete

pipboy96 added the nsfw label Oct 2, 2019

EFForg / https-everywhere

Add some imageboard domains #18038

Add some imageboard domains #18038

pipboy96 commented May 30, 2019 •

edited

EFForg / https-everywhere

Join GitHub today

Add some imageboard domains #18038

Add some imageboard domains #18038

Comments

pipboy96 commented May 30, 2019 • edited

Alexa top 1M

The rest

The script

pipboy96 commented May 30, 2019 •

edited