[pacman-mirrors-dev] Testing mirrors async

Pacman-mirrors has been cooking and version 4.15.0.dev7 is available from unstable branch with an experimental async feature.

The -f has always excluded mirrors which were not up-to-date but sometimes mirrors have other issues and those is attempted filtered as well.

Black-hole mirrors - mirrors which for some reason does not respond to the python webclient - are also filtered.

mirror check

As always your feedback is highly valued, so please test it and post your results - the good as well as the bad.


One known issue - your branch may change to stable - that is the installer.

pacman-mirrors -G

You can revert to unstable using

sudo pacman-mirrors -aS unstable

Change Log

All notable changes to this project will be documented in this file.

[4.15.0] 2019-07-13

  • Refactor internal mirrorpool building - excluding mirros which unresponsive or not up-to-date
  • Experimental argument --test-async
    @ZenTauro code added for #144
    Overall functionality works - needs extra work before full implementation.

[4.14.99.dev] 2019-04-25

  • Implemented #146
    • Added --interval. Works only with --no-status filtering mirrors based on last sync time.
  • Fixed #145
5 Likes

Very good Update :+1:

for test, i make a go script for test only speed

go because is the best for sync (it's easier than python). Result is very fast with all 90 url, and no blocking :slight_smile:


import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	url_status string = "https://repo.manjaro.org/status.json"
	//https://misc.flogisoft.com/bash/tip_colors_and_formatting
	COLOR_NONE  = "\033[0m"
	COLOR_BLUE  = "\033[0;34m"
	COLOR_GREEN = "\033[0;36m"
	COLOR_RED   = "\033[38;5;124m"
	COLOR_GRAY  = "\033[38;5;243m"
	_VERSION    = "0.0.4"
)

/*
   user values
*/

// input values
type Options struct {
	sync, help, countries, async bool
	country                      string
	limit, timeout               uint64
}

// parse console input values
func argsParser() Options {
	ret := Options{
		async:   true,
		limit:   8,
		timeout: 10,
	}
	limit, err := strconv.ParseUint(os.Getenv("LIMIT"), 10, 8)
	if err != nil || limit < 1 {
		ret.limit = 8
	} else {
		ret.limit = limit
	}
	timeout, err := strconv.ParseUint(os.Getenv("TIMEOUT"), 10, 8)
	if err != nil || timeout < 1 {
		ret.timeout = 10
	} else {
		ret.timeout = timeout
	}

	for _, arg := range os.Args[1:] {
		if len(arg) < 1 {
			// user enter empty values : program.go "" "" "" !
			continue
		}
		if !strings.HasPrefix(arg, "-") {
			// first is country filter
			ret.country = strings.ToLower(arg)
			continue
		}
		// test options
		for _, ch := range arg[1:] {
			switch ch {
			case 'h':
				ret.help = true // TODO usage()
			case 's':
				ret.sync = true
			case 'n':
				ret.async = false
			case 'c':
				ret.countries = true // TODO display list countries
			}
		}
	}

	return ret
}

/*
   JSON structure
*/

// generate with https://mholt.github.io/json-to-go/
type Mirror struct {
	Status    [3]int   `json:"branches"`
	Country   string   `json:"country"`
	LastSync  string   `json:"last_sync"` // can empty
	Protocols []string `json:"protocols"`
	URL       string   `json:"url"`
	id        int
	speed     time.Duration
}

func (m *Mirror) stable() bool {
	return m.Status[0] == 1
}
func (m *Mirror) testing() bool {
	return m.Status[1] == 1
}
func (m *Mirror) unstable() bool {
	return m.Status[2] == 1
}
func (m *Mirror) isBreak() bool {
	return m.Status[0] < 0
}
func (m *Mirror) isSync() bool {
	for _, value := range m.Status {
		if value != 1 {
			return false
		}
	}
	return true
}
func (m *Mirror) sortSpeed() time.Duration {
	if m.speed < 1 {
		return 99999999999
	}
	return m.speed
}

// test mirror speed
func (m *Mirror) testSpeed(timeout uint64) (time.Duration, error) {
	m.speed = -1
	path := "/tmp/mirrors-test"
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.Mkdir(path, 0700)
	}
	path = fmt.Sprintf("%v/%v", path, m.id)
	url := m.URL + "stable/core/x86_64/core.db.tar.gz"

	client := http.Client{Timeout: time.Duration(timeout) * time.Second}
	resp, err := client.Get(url)
	if err != nil {
		return -2, err
	}
	defer resp.Body.Close()
	if resp.StatusCode > 399 {
		return -3, fmt.Errorf("http error: %v", resp.StatusCode)
	}
	//fmt.Println(resp)

	// Create file
	out, err := os.Create(path)
	if err != nil {
		return -1, err
	}
	defer out.Close()

	start := time.Now() // start timer

	// read file and write datas in tmp file
	_, err = io.Copy(out, resp.Body)

	// test size ...
	size, _ := strconv.Atoi(resp.Header.Get("Content-Length")) // bad with some servers ! too small
	downloadSize := int64(size)
	fstat, err := out.Stat()
	if fstat.Size() > downloadSize {
		// timer has expired, we have not all datas
		m.speed = -4
		return m.speed, fmt.Errorf("Bad file sizes %v > %v (downloaded)", fstat.Size(), downloadSize)
	}

	defer func() {
		elapsed := time.Since(start)
		m.speed = elapsed
	}()

	return m.speed, nil
}

func (m *Mirror) displayStatus() string {
	fields := [3]string{"S", "T", "U"}
	for i, id := range fields {
		color := COLOR_GREEN
		if m.Status[i] != 1 {
			color = COLOR_RED
		}
		fields[i] = fmt.Sprintf("%v%v%v", color, id, COLOR_NONE)
	}
	return strings.Join(fields[:], " ")
}

func (m Mirror) String() string {
	var dur string
	if m.speed < 1 {
		dur = fmt.Sprintf("%v%v%v %v", COLOR_RED, int64(m.speed), COLOR_NONE, m.Status)
	} else {
		dur = fmt.Sprintf("%v%v%v", COLOR_GREEN, m.speed, COLOR_NONE)
	}
	return fmt.Sprintf("%v %-17v %5v %-61v %v (%v) %s", m.displayStatus(), m.Country, m.LastSync, m.URL, m.Protocols, m.id, dur)
}

type Mirrors []Mirror

// download manjaro json, return slice struct of all mirrors
func downloadDatas(wantSync bool, wantCountry string) (Mirrors, error) {
	resp, err := http.Get(url_status)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var datas Mirrors
	json.Unmarshal([]byte(body), &datas)

	var ret Mirrors
	for id, _ := range datas {
		datas[id].id = id
		// set filters
		if wantSync && !datas[id].isSync() {
			continue
		}
		if wantCountry != "" && strings.ToLower(datas[id].Country) != wantCountry {
			continue
		}
		ret = append(ret, datas[id])
	}

	sort.Slice(ret, func(i, j int) bool {
		return ret[i].Country < ret[j].Country
	})

	return ret, nil
}

// display coutries list and exit
func displayContries(mirrors *Mirrors) {
	countries := func() []string {
		rets := []string{}

		for _, s := range *mirrors {
			if !func(c string) bool {
				for _, value := range rets {
					if value == c {
						return true
					}
				}
				return false
			}(s.Country) {
				rets = append(rets, s.Country)
			}
		}
		return rets
	}()
	fmt.Println(countries)
	os.Exit(0)
}

func usage(options *Options) {
	fmt.Printf("Usage: %v [Country_Nane]\n", filepath.Base(os.Args[0]))
	fmt.Printf("  %-15v: this help\n", "-h")
	fmt.Printf("  %-15v: only updated mirrors [1 1 1]\n", "-s")
	fmt.Printf("  %-15v: countries list\n", "-c")
	fmt.Printf("  %-15v: no async !\n", "-n")
	fmt.Printf("  %-15v: optional - country filter\n", "[Country_Nane]")
	fmt.Printf("\nenv LIMIT=%v (allowed to run concurrently)\n", options.limit)
	fmt.Printf("\nenv TIMEOUT=%v http request timeout\n", options.timeout)
	os.Exit(0)
}

/*
   main Application
*/

func main() {
	println("mirrors manjaro...")

	options := argsParser()

	var mirrors Mirrors
	mirrors, err := downloadDatas(options.sync, options.country)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(2)
	}
	println("")

	if options.help {
		usage(&options)
	}

	if options.countries {
		displayContries(&mirrors)
	}

	// for info display list
	fmt.Println(len(mirrors), "Manjaro mirrors")
	for _, mirror := range mirrors {
		fmt.Println(mirror)
	}
	println("")
	//os.Exit(0)

	/*
	   async test speed
	*/

	type Message struct {
		id  int
		msg string
	}

	var waitg sync.WaitGroup
	var mutex = sync.Mutex{}
	sem := make(chan bool, 16) // limit async

	println("Speed tests ...")
	start := time.Now() // start timer
	for id, _ := range mirrors[:] {
		waitg.Add(1)
		go func(id int) {
			if options.async {
				sem <- true
				defer func() { <-sem }()
			}
			m := mirrors[id]
			defer waitg.Done()
			_, err := m.testSpeed(options.timeout)
			if err != nil {
				fmt.Println(m.id, COLOR_RED, err.Error(), COLOR_NONE)
			}
			mutex.Lock()
			mirrors[id] = m
			mutex.Unlock()
			fmt.Println("Duration:", m.speed, m.Country, m.URL, m.id)
			if !options.async {
				sem <- true
			}
		}(id)
		if !options.async {
			<-sem
		}
	}
	waitg.Wait()
	close(sem)
	fmt.Println("\n:: run all test : ", time.Since(start), "\n")

	sort.Slice(mirrors, func(i, j int) bool {
		return mirrors[i].sortSpeed() < mirrors[j].sortSpeed()
	})

	println("Sorted:")
	for _, mirror := range mirrors {
		fmt.Println(mirror)
	}
}

result:

:: run all test :  9.843313501s
Sorted:
S T U United_Kingdom    02:34 http://mirror.catn.com/pub/manjaro/                           [http] (80) 151.593288ms
S T U Belgium           77:37 http://ftp.belnet.be/mirrors/manjaro/                         [http ftp] (5) 164.491496ms
S T U Poland            00:30 https://mirror.tuchola-dc.pl/manjaro/                         [https http] (65) 208.456138ms
...
S T U Brazil            52:41 http://mirror.ufam.edu.br/manjaro/                            [http] (10) 2.416762552s
S T U Japan             08:00 http://ftp.tsukuba.wide.ad.jp/Linux/manjaro/                  [http] (57) 2.444424572s
S T U South_Africa      16:03 http://manjaro.mirror.ac.za/                                  [http ftp] (71) 2.600995234s
...
S T U Brazil                  https://www.caco.ic.unicamp.br/manjaro/                       [https http] (11) -1
S T U Denmark           00:02 https://www.uex.dk/public/manjaro/                            [https] (27) -4
S T U Brazil                  http://linorg.usp.br/manjaro/                                 [http] (9) -1
S T U Brazil                  http://pet.inf.ufsc.br/mirrors/manjarolinux/                  [http] (8) -1
S T U Indonesia               http://kartolo.sby.datautama.net.id/manjaro/                  [http] (48) -1
S T U Spain                   https://ftp.caliu.cat/pub/distribucions/manjaro/              [https http ftp] (73) -1

last field : duration miliseconds ( or <0 if error)
core.db in /tmp/mirrors-test, this directory not removed at end !

with a file main.go
evaluate : go run main.go
build executable : go build -ldflags "-s"

2 Likes

@linux-aarhus

in fact after tests with this tools, I realize that testing the speed asynchronously is a bad idea for pacman-mirrors :

  • If we have a small internet speed

When we test the eleventh url (very fast mirror), we download 10 files at the same time and so the eleventh is very slow by only our Internet service provider and not by mirror server :cry: Or the reverse, an url can have 5 timeouts before it and it's paradise for him. The ranking no longer makes sense! :upside_down_face:

if a "pro web" @codesardine can confirm ?


new version (mv channel sync/async)
v0.0.4 : option -n for no async http get

async: 17seconds -->> sync : 2mn 15 seconds :nauseated_face:

result sorted differents (same no async)

:: run all test :  2m4.168284236s 
S T U France            00:12 https://manjaro.ynh.ovh/                                      [https] (31) 73.769509ms
S T U France            02:12 https://mirror.oldsql.cc/manjaro/                             [https] (30) 75.102205ms
S T U Germany           00:03 https://mirror.philpot.de/manjaro/                            [http https] (43) 77.402815ms
S T U Italy             02:13 https://ct.mirror.garr.it/mirrors/manjaro/                    [https] (55) 82.596174ms

: run all test :  2m21.070828486s 
Sorted:
S T U France            00:12 https://manjaro.ynh.ovh/                                      [https] (31) 72.463111ms
S T U France            02:12 https://mirror.oldsql.cc/manjaro/                             [https] (30) 75.384915ms
S T U Germany           00:03 https://mirror.philpot.de/manjaro/                            [http https] (43) 76.879401ms
S T U Italy             02:13 https://ct.mirror.garr.it/mirrors/manjaro/                    [https] (55) 82.88946ms

:: run all test :  15.334460458s 
S T U France            07:52 https://mirror.oldsql.cc/manjaro/                             [https] (30) 98.090705ms
S T U Belgium           83:58 http://ftp.belnet.be/mirrors/manjaro/                         [http ftp] (5) 118.578916ms
S T U Germany                 http://ftp.rz.tu-bs.de/pub/mirror/manjaro.org/repos/          [ftp http] (42) 131.551589ms
S T U Hungary           00:23 https://quantum-mirror.hu/mirrors/pub/manjaro/                [https http] (47) 139.408583ms

: run all test :  16.460254609s 
S T U Germany           00:03 https://mirror.philpot.de/manjaro/                            [http https] (43) 124.225421ms
S T U Netherlands       00:40 https://mirror.koddos.net/manjaro/                            [https] (62) 127.438631ms
S T U Netherlands       00:43 https://ftp.nluug.nl/pub/os/Linux/distr/manjaro/              [ftp https] (60) 133.315307ms
S T U Russia            01:03 https://mirror.yandex.ru/mirrors/manjaro/                     [https] (68) 154.68231ms
S T U United_Kingdom    00:34 http://repo.manjaro.org.uk/                                   [http] (81) 160.511972ms

conclusion ? async call gives false results
(here, async or not we have exactly the same code - only the line "<-sem" is moved for wait download end)

1 Like

Thank you for investigating this. I appreciate your efforts very much. I like your thinking, which contradicts mine and thus we getter a better result. Thank you.

I have been speculating what may cause the lockups and am working towards the content type checking - size of - due to your results when experimenting with go.

What have been confusing to me is the apparent similarity when comparing the results - between the standard loop vs. the threading fetch.

Could it be that the concurrent threads would actually be looking at the same file and thus the accumulating responses being a factor in having the wrong results?

1 Like

not same file, here i set a id at all url and create a file in /tmp by url(id)

ls -l /tmp/mirrors-test
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 0
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 1
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 10
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 12
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 13
...
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 85
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 86
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 87
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 88
-rw-r--r-- 1 patrick users 149K "06.08.2019 13:01" 89
-rw-r--r-- 1 patrick users 149K "06.08.2019 12:56" 9

as write at top, for me result is false if we have not the bandwith for download "X" files :

  • if my bandwith can't download 10 files at same time : i mesure my bandwith and not server speed - for exemple, with 7 downloads, i use all my bandwith : url "8" will be slow even if it's my fastest mirror
  • in a thread of 10 download (at same time), i can have 8 timeout : so i have a big bandwith for only 2 download (ok is extrem but is for image)

I understand what you are saying - more simultanous downloads will decrease the overall available bandwith and therefore messing with the results.

Which returns us to the current loop model which probes one mirror at a time.

On another note:

If we are running simultanous concurrent downloads on a specific file - the downloads would be equally handicapped by the concurrent downloads and - in the end the result would still - somehow - reflect the availability for that current user.

And the availability - no matter how crowded - is the goal of the probing. Am I being clear?

not sure :slight_smile: in my "bad" exemple, if my best mirror is the id 8, it will not be recognized as a fast server. So ranking is false. user have a "middle" bandwith and result is false

have less thread is good ? (we have 50 ; me with this tools only 16...) with pacman-mirrors) only if mirrors have not timeouts : with 4 thread, the risk is to have 3 timeout and only one download : pacman-mirrors can view this mirror as the best speed ...

I am not familiar with the pacman-mirrors code base.

yes but here is to understand if async call return good or not result ?

pacman-mirrors make async call GET to core.db et return times for 90 url

Threaded testing of sites doesn't make much sense, so how about a different approach to the tests?

Test the mirrors in a series of rounds/passes with increasing timeouts:

First round: time out after two seconds
Second round: time out after four seconds
Third round: time out after eight seconds
etc.

Continue testing mirrors until you have the target number of mirrors.

This should yield a result quickly for fast connections (i.e. quickly skip any but the fastest mirrors) but allow slower connections to still find appropriate mirrors.

In pacman-mirrors there is not good or bad.

It is entirely judged by the availability of a specific - always available file - and the time spend to completely retrieve this file which depends solely on the executors system and internet connection wheter that is directly, NAT or through proxy.

This is entirely different depending on where in the world the executor is placed and whatever time is used downloading the file.

If that file is retrieved simultanously through 10 threads then the final result will reflect that fact. And the result will reflect that one mirror will be faster over another.

I agree there may be a caching issue using the experimental argument - but as far the tests as revealed little difference between the current loop-method vs the async method.

But then again I may be wrong.

What I can say is that downloading in parallel will always be faster independently of available bandwidth.

YES YES YES
my tool make both with same code 16 seconds asyn for 2 minutes sync :smile: (90 urls with 10 seconds timeout)


yes for purpose of @jonathon
firsts pass async small files (bandwith is not overloaded) and last pass sync for real time

I make a test with state file 256B, timeout 2s

: run all test :  4.005255414s 
S T U Germany           00:10 https://mirror.23media.com/manjaro/                           [https http] (36) 37.452µs
...
S T U Iran              04:50 https://repo.sadjad.ac.ir/manjaro/                            [https http] (50) 200.844349ms

here the problem is with 90 url, if user chose one country i suppose that bandwith is not overloaded with 2..5 downloads

1 Like

The Iranian mirrors is the culprit on every run.

I do think it is related to their internet censoring - but I have really no idea.

it's only python :wink:

with go no problem :slight_smile: At first pass

Duration: 78.425µs Iran https://repo.iut.ac.ir/repo/manjaro/ 51
Duration: 104.311µs Iran https://repo.iut.ac.ir/repo/manjaro/ 52
Duration: 199.625699ms Iran https://repo.sadjad.ac.ir/manjaro/ 50

!!! Error in json we have twice repo.iut.ac.ir

BUT we can view big speed difference for same server with async : 78 != 104 :cry:
...and other run it's good, same speed

Duration: 119.067µs Iran https://repo.iut.ac.ir/repo/manjaro/ 51
Duration: 120.044µs Iran https://repo.iut.ac.ir/repo/manjaro/ 52

tool rewrited with multi-pass - but not sure to have good results :

5 pass async but if count < 12 use call sync
At all pass remove 2/3 after sort by speed
break if count < 6

finish with a "long" test sync (before was a async 2 seconds)

:: run all test :  5.056785018s
5 Sorted:
S T U Denmark           02:19 https://mirrors.dotsrc.org/manjaro/                           [https] (26) 116.962465ms
S T U France            00:33 https://manjaro.ynh.ovh/                                      [https] (31) 146.320054ms
S T U Germany           00:03 https://manjaro.moson.eu/                                     [http https] (44) 164.327006ms
S T U Germany           01:30 https://mirror.23media.com/manjaro/                           [https http] (36) 190.033618ms
S T U United_Kingdom    00:11 http://manjaro.mirrors.uk2.net/                               [http] (83) 202.053098ms

package main

import (
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	url_status string = "https://repo.manjaro.org/status.json"
	//https://misc.flogisoft.com/bash/tip_colors_and_formatting
	COLOR_NONE  = "\033[0m"
	COLOR_BLUE  = "\033[0;34m"
	COLOR_GREEN = "\033[0;36m"
	COLOR_RED   = "\033[38;5;124m"
	COLOR_GRAY  = "\033[38;5;243m"
	_VERSION    = "0.0.5"
)

/*
   user values
*/

// input values
type Options struct {
	sync, help, countries, async, onepass bool
	country                               string
	limit, timeout                        uint64
}

// parse console input values
func argsParser() Options {
	ret := Options{
		async:   true,
		limit:   8,
		timeout: 10,
	}
	limit, err := strconv.ParseUint(os.Getenv("LIMIT"), 10, 8)
	if err != nil || limit < 1 {
		ret.limit = 8
	} else {
		ret.limit = limit
	}
	timeout, err := strconv.ParseUint(os.Getenv("TIMEOUT"), 10, 8)
	if err != nil || timeout < 1 {
		ret.timeout = 10
	} else {
		ret.timeout = timeout
	}

	for _, arg := range os.Args[1:] {
		if len(arg) < 1 {
			// user enter empty values : program.go "" "" "" !
			continue
		}
		if !strings.HasPrefix(arg, "-") {
			// first is country filter
			ret.country = strings.ToLower(arg)
			continue
		}
		// test options
		for _, ch := range arg[1:] {
			switch ch {
			case 'h':
				ret.help = true // TODO usage()
			case 's':
				ret.sync = true
			case 'n':
				ret.async = false
			case '1':
				ret.onepass = true
			case 'c':
				ret.countries = true // TODO display list countries
			}
		}
	}

	return ret
}

/*
   JSON structure
*/

// generate with https://mholt.github.io/json-to-go/
type Mirror struct {
	Status    [3]int   `json:"branches"`
	Country   string   `json:"country"`
	LastSync  string   `json:"last_sync"` // can empty
	Protocols []string `json:"protocols"`
	URL       string   `json:"url"`
	id        int
	speed     time.Duration
}

func (m *Mirror) stable() bool {
	return m.Status[0] == 1
}
func (m *Mirror) testing() bool {
	return m.Status[1] == 1
}
func (m *Mirror) unstable() bool {
	return m.Status[2] == 1
}
func (m *Mirror) isBreak() bool {
	return m.Status[0] < 0
}
func (m *Mirror) isSync() bool {
	for _, value := range m.Status {
		if value != 1 {
			return false
		}
	}
	return true
}
func (m *Mirror) sortSpeed() time.Duration {
	if m.speed < 1 {
		return 99999999999
	}
	return m.speed
}

// test mirror speed
func (m *Mirror) testSpeed(timeout uint64, file string) (time.Duration, error) {
	m.speed = -1
	path := "/tmp/mirrors-test"
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.Mkdir(path, 0700)
	}
	path = fmt.Sprintf("%v/%v", path, m.id)
	//url := m.URL + "stable/core/x86_64/core.db.tar.gz"
	url := m.URL + file //"stable/state"

	client := http.Client{Timeout: time.Duration(timeout) * time.Second}
	resp, err := client.Get(url)
	if err != nil {
		return -2, err
	}
	defer resp.Body.Close()
	if resp.StatusCode > 399 {
		return -3, fmt.Errorf("http error: %v", resp.StatusCode)
	}
	//fmt.Println(resp)

	// Create file
	out, err := os.Create(path)
	if err != nil {
		return -1, err
	}
	defer out.Close()

	start := time.Now() // start timer

	// read file and write datas in tmp file
	_, err = io.Copy(out, resp.Body)

	// test size ...
	size, _ := strconv.Atoi(resp.Header.Get("Content-Length")) // bad with some servers ! too small
	downloadSize := int64(size)
	fstat, err := out.Stat()
	if fstat.Size() > downloadSize {
		// timer has expired, we have not all datas
		m.speed = -4
		return m.speed, fmt.Errorf("Bad file sizes %v > %v (downloaded)", fstat.Size(), downloadSize)
	}

	defer func() {
		elapsed := time.Since(start)
		m.speed = elapsed
	}()

	return m.speed, nil
}

func (m *Mirror) displayStatus() string {
	fields := [3]string{"S", "T", "U"}
	for i, id := range fields {
		color := COLOR_GREEN
		if m.Status[i] != 1 {
			color = COLOR_RED
		}
		fields[i] = fmt.Sprintf("%v%v%v", color, id, COLOR_NONE)
	}
	return strings.Join(fields[:], " ")
}

func (m Mirror) String() string {
	var dur string
	if m.speed < 1 {
		dur = fmt.Sprintf("%v%v%v %v", COLOR_RED, int64(m.speed), COLOR_NONE, m.Status)
	} else {
		dur = fmt.Sprintf("%v%v%v", COLOR_GREEN, m.speed, COLOR_NONE)
	}
	return fmt.Sprintf("%v %-17v %5v %-61v %v (%v) %s", m.displayStatus(), m.Country, m.LastSync, m.URL, m.Protocols, m.id, dur)
}

type Mirrors []Mirror

// download manjaro json, return slice struct of all mirrors
func downloadDatas(wantSync bool, wantCountry string) (Mirrors, error) {
	resp, err := http.Get(url_status)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var datas Mirrors
	json.Unmarshal([]byte(body), &datas)

	var ret Mirrors
	for id, _ := range datas {
		datas[id].id = id
		// set filters
		if wantSync && !datas[id].isSync() {
			continue
		}
		if wantCountry != "" && strings.ToLower(datas[id].Country) != wantCountry {
			continue
		}
		ret = append(ret, datas[id])
	}

	sort.Slice(ret, func(i, j int) bool {
		return ret[i].Country < ret[j].Country
	})

	return ret, nil
}

// display coutries list and exit
func displayContries(mirrors Mirrors) {
	countries := func() []string {
		rets := []string{}

		for _, s := range mirrors {
			if !func(c string) bool {
				for _, value := range rets {
					if value == c {
						return true
					}
				}
				return false
			}(s.Country) {
				rets = append(rets, s.Country)
			}
		}
		return rets
	}()
	fmt.Println(countries)
	os.Exit(0)
}

func logResults(mirrors Mirrors, options Options) {
	lfile, err := os.OpenFile("mirrors.log", os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0640)
	if err != nil {
		fmt.Println("Failed to open log file results.log:", err)
		return
	}
	defer lfile.Close()
	for i, mirror := range mirrors {
		//lfile.WriteString(text)
		fmt.Fprintf(lfile, "%-2v\t", mirror.id)
		if i > 4 {
			break
		}
	}
	fmt.Fprintln(lfile, options)
}

func usage(options *Options) {
	fmt.Printf("Usage: %v [Country_Nane]\n", filepath.Base(os.Args[0]))
	fmt.Printf("  %-15v: this help\n", "-h")
	fmt.Printf("  %-15v: only updated mirrors [1 1 1]\n", "-s")
	fmt.Printf("  %-15v: countries list\n", "-c")
	fmt.Printf("  %-15v: no async !\n", "-n")
	fmt.Printf("  %-15v: one pass\n", "-1")
	fmt.Printf("  %-15v: optional - country filter\n", "[Country_Nane]")
	fmt.Printf("\nenv LIMIT=%v (allowed to run concurrently)\n", options.limit)
	fmt.Printf("\nenv TIMEOUT=%v http request timeout\n", options.timeout)
	os.Exit(0)
}

/*
   main Application
*/

func main() {
	println("mirrors manjaro...")

	options := argsParser()

	var mirrors Mirrors
	mirrors, err := downloadDatas(options.sync, options.country)
	if err != nil {
		fmt.Println(err.Error())
		os.Exit(2)
	}
	println("")

	if options.help {
		usage(&options)
	}

	if options.countries {
		displayContries(mirrors)
	}

	// for info display list
	fmt.Println(len(mirrors), "Manjaro mirrors")
	for _, mirror := range mirrors {
		fmt.Println(mirror)
	}
	println("")
	//os.Exit(0)

	/*
	   async test speed
	*/
	var rounds []string

	rounds = []string{
		"stable/state",
		"stable/core/x86_64/core.db.tar.gz",
		"stable/core/x86_64/core.db.tar.gz",
		"stable/core/x86_64/core.db.tar.gz",
		"stable/core/x86_64/core.db.tar.gz",
	}
	if options.onepass {
		// not rounds is a other var !
		rounds = []string{rounds[4]}
	}

	for i, round := range rounds {

		timeout := options.timeout
		if !options.onepass {
			options.timeout = (uint64(i) + 1) * 2
			//round = "stable/core/x86_64/core.db.tar.gz"
		}
		passTest(mirrors, options, i, round)
		options.timeout = timeout
		mirrors = passFilter(mirrors)

		println(len(mirrors), "Sorted:")
		for _, mirror := range mirrors {
			fmt.Println(mirror)
		}
		if len(mirrors) < 12 {
			options.async = false
		}
		if len(mirrors) < 6 {
			break
		}
	}
	logResults(mirrors, options)
}

func passTest(mirrors Mirrors, options Options, count int, file string) {
	var waitg sync.WaitGroup
	var mutex = sync.Mutex{}
	sem := make(chan bool, 16) // limit async

	println("Speed tests ...")
	start := time.Now() // start timer
	for id, _ := range mirrors[:] {
		waitg.Add(1)
		go func(id int) {
			if options.async {
				sem <- true
				defer func() { <-sem }()
			}
			m := mirrors[id]
			defer waitg.Done()
			_, err := m.testSpeed(options.timeout, file)
			if err != nil {
				fmt.Println(m.id, COLOR_RED, err.Error(), COLOR_NONE)
			}
			mutex.Lock()
			mirrors[id] = m
			mutex.Unlock()
			if count == 0 {
				fmt.Println("Duration:", m.speed, m.Country, m.URL, m.id)
			}
			if !options.async {
				sem <- true
			}
		}(id)
		if !options.async {
			<-sem
		}
	}
	waitg.Wait()
	close(sem)
	fmt.Println("\n:: run all test : ", time.Since(start))
}

// after one test remove low speeds
func passFilter(mirrors Mirrors) Mirrors {
	sort.Slice(mirrors, func(i, j int) bool {
		return mirrors[i].sortSpeed() < mirrors[j].sortSpeed()
	})
	i := 2
	if len(mirrors) < 2 {
		i = 1
	}
	return mirrors[0 : len(mirrors)/i]
}

Last version (0.0.5) add logs

only id of url for easy compare - last colum is user parameters for sort lines

Result ... async is crasy , some pass with asyn is not best solution (remove 50% at all pass and sync at last)

run (re-run) same parameters in 5..10 minutes !
tests with a 16Mb/s bandwith

sync - 4 pass (-n -s)
31	30	33	43	{true false false false false  8 10}	
31	30	33	43	{true false false false false  8 10}
30	31	43	38	{true false false false false  8 10}
30	31	43	33	{true false false false false  8 10}

sync - one pass (-n -s -1 )
31	30	33	83	43	82	{true false false false true  8 10}
31	30	33	43	83	82	{true false false false true  8 10}

async - one pass (-1)
36	43	31	83	60	44	{true false false true true  8 10}
31	82	83	26	30	59	{true false false true true  8 10}
31	25	68	36	26	82	{true false false true true  8 10}
36	68	82	33	59	74	{true false false true true  8 10}
30	68	6 	82	38	83	{true false false true true  8 10}
30	25	26	61	14	31	{true false false true true  8 10}

async - 4 pass (last sync) (no parameters)
82	43	26	1 	61	{false false false false false  8 10}
25	30	83	31	80	{false false false false false  8 10}
26	83	39	1 	63	{false false false false false  8 10}
26	60	32	37	31	{false false false false false  8 10}
60	25	32	26	30	{false false false false false  8 10}
25	26	30	80	82	{false false false false false  8 10}
26	60	30	36	31	{false false false false false  8 10}

async - one pass is not catastrophic (30 and 31 are present) but is not constant ! not sure to use the best server for me but only generally one of the best
ps: 30 and 31 my country mirrors


ps: i use "only" 8 thread async , pacman-mirrors 50

with 32, done in 5 minutes (with 8 thread I could see 30 and 31 but here ?)

async - one pass (-1) but with 32 thread
25	81	82	80	45	65	{false false false true true  32 10}
82	53	80	31	25	60	{false false false true true  32 10}
31	76	25	68	26	83	{false false false true true  32 10}
53	80	25	65	76	39	{false false false true true  32 10}
25	76	31	32	44	83	{false false false true true  32 10}
25	26	31	30	82	76	{false false false true true  32 10}
and one pass sync after for compare
31	82	60	25	32	76	{false false false false true  8 10}

it would be good to have the same logs in pacman-mirrors to see if results are constant or random.
if a mirror wins or loses a place it's okay, but otherwise the app doesn't do all its job

1 Like

last test, create a custom list : 30 same url

nosync :

Duration: 87.424589ms Germany https://mirror.alpix.eu/manjaro/ 0
Duration: 76.596148ms Germany https://mirror.alpix.eu/manjaro/ 1
Duration: 75.35862ms Germany https://mirror.alpix.eu/manjaro/ 2
Duration: 74.938965ms Germany https://mirror.alpix.eu/manjaro/ 3
Duration: 74.860048ms Germany https://mirror.alpix.eu/manjaro/ 4
Duration: 74.778651ms Germany https://mirror.alpix.eu/manjaro/ 5
Duration: 75.19715ms Germany https://mirror.alpix.eu/manjaro/ 6
Duration: 81.683102ms Germany https://mirror.alpix.eu/manjaro/ 7
Duration: 76.784249ms Germany https://mirror.alpix.eu/manjaro/ 8
Duration: 75.77322ms Germany https://mirror.alpix.eu/manjaro/ 9
Duration: 74.762405ms Germany https://mirror.alpix.eu/manjaro/ 10
Duration: 75.26492ms Germany https://mirror.alpix.eu/manjaro/ 11
Duration: 75.357106ms Germany https://mirror.alpix.eu/manjaro/ 12
Duration: 74.55967ms Germany https://mirror.alpix.eu/manjaro/ 13
Duration: 75.159842ms Germany https://mirror.alpix.eu/manjaro/ 14
Duration: 75.601388ms Germany https://mirror.alpix.eu/manjaro/ 15
Duration: 74.978447ms Germany https://mirror.alpix.eu/manjaro/ 16
Duration: 74.595456ms Germany https://mirror.alpix.eu/manjaro/ 17
Duration: 74.981465ms Germany https://mirror.alpix.eu/manjaro/ 18
Duration: 79.129708ms Germany https://mirror.alpix.eu/manjaro/ 19
Duration: 76.053345ms Germany https://mirror.alpix.eu/manjaro/ 20
Duration: 75.205484ms Germany https://mirror.alpix.eu/manjaro/ 21
Duration: 75.175662ms Germany https://mirror.alpix.eu/manjaro/ 22
Duration: 75.413898ms Germany https://mirror.alpix.eu/manjaro/ 23
Duration: 75.488194ms Germany https://mirror.alpix.eu/manjaro/ 24
Duration: 74.751576ms Germany https://mirror.alpix.eu/manjaro/ 25
Duration: 74.605584ms Germany https://mirror.alpix.eu/manjaro/ 26
Duration: 75.184611ms Germany https://mirror.alpix.eu/manjaro/ 27
Duration: 74.955526ms Germany https://mirror.alpix.eu/manjaro/ 28
Duration: 74.852167ms Germany https://mirror.alpix.eu/manjaro/ 29

async one pass (id - 17,7,4,...) is order in list
clearly not same time (not server/internet but my manjaro is guilty) and speed range is too big : 507ms to 1.3sec

15 Sorted:
Duration: 507.661912ms Germany https://mirror.alpix.eu/manjaro/ 17
Duration: 1.193529959s Germany https://mirror.alpix.eu/manjaro/ 7
Duration: 1.212192495s Germany https://mirror.alpix.eu/manjaro/ 4
Duration: 1.220565606s Germany https://mirror.alpix.eu/manjaro/ 2
Duration: 1.229723762s Germany https://mirror.alpix.eu/manjaro/ 6
Duration: 1.24754756s Germany https://mirror.alpix.eu/manjaro/ 29
Duration: 1.257017259s Germany https://mirror.alpix.eu/manjaro/ 19
Duration: 1.266215957s Germany https://mirror.alpix.eu/manjaro/ 16
Duration: 1.284159775s Germany https://mirror.alpix.eu/manjaro/ 3
Duration: 1.293196608s Germany https://mirror.alpix.eu/manjaro/ 0
Duration: 1.301707901s Germany https://mirror.alpix.eu/manjaro/ 9
Duration: 1.281956719s Germany https://mirror.alpix.eu/manjaro/ 20
Duration: 1.290178803s Germany https://mirror.alpix.eu/manjaro/ 10
Duration: 1.299592943s Germany https://mirror.alpix.eu/manjaro/ 5
Duration: 1.344692015s Germany https://mirror.alpix.eu/manjaro/ 1
Duration: 1.362789183s Germany https://mirror.alpix.eu/manjaro/ 8
Duration: 953.36103ms Germany https://mirror.alpix.eu/manjaro/ 26
Duration: 617.925255ms Germany https://mirror.alpix.eu/manjaro/ 21
Duration: 844.423725ms Germany https://mirror.alpix.eu/manjaro/ 22
Duration: 962.441538ms Germany https://mirror.alpix.eu/manjaro/ 23
Duration: 980.52684ms Germany https://mirror.alpix.eu/manjaro/ 24
Duration: 989.587424ms Germany https://mirror.alpix.eu/manjaro/ 25
Duration: 999.109261ms Germany https://mirror.alpix.eu/manjaro/ 28
Duration: 1.01619544s Germany https://mirror.alpix.eu/manjaro/ 27
Duration: 1.026031688s Germany https://mirror.alpix.eu/manjaro/ 18
Duration: 961.687612ms Germany https://mirror.alpix.eu/manjaro/ 13
Duration: 970.978747ms Germany https://mirror.alpix.eu/manjaro/ 11
Duration: 988.890395ms Germany https://mirror.alpix.eu/manjaro/ 14
Duration: 998.119425ms Germany https://mirror.alpix.eu/manjaro/ 12
Duration: 1.005668199s Germany https://mirror.alpix.eu/manjaro/ 15
1 Like

Thank you so much for testing this.

Apparently there is a difference between sync and async using Python and Go.

The results using Python sync/async are slightly different but not that much off.

  • 300Mbit fibre optic connection
  • unstable branch
  • -c Germany

First a sync - default pacman-mirrors test

~ >>> sudo pacman-mirrors -c Germany
[sudo] password for fh: 
::INFO Downloading mirrors from repo.manjaro.org
::INFO User generated mirror list
::------------------------------------------------------------
::INFO Custom mirror file saved: /var/lib/pacman-mirrors/custom-mirrors.json
::INFO Using custom mirror file
::INFO Querying mirrors - This may take some time
  0.172 Germany        : https://mirror.alpix.eu/manjaro/
  0.175 Germany        : https://mirror.23media.com/manjaro/
  0.123 Germany        : http://mirror.23media.com/manjaro/
  0.168 Germany        : http://mirror.ragenetwork.de/manjaro/
  0.235 Germany        : https://mirror.netzspielplatz.de/manjaro/packages/
  0.169 Germany        : https://mirror.philpot.de/manjaro/
  0.143 Germany        : http://mirror.philpot.de/manjaro/
  0.153 Germany        : https://manjaro.moson.eu/
  0.133 Germany        : http://manjaro.moson.eu/
::INFO Writing mirror list
::Germany         : https://mirror.23media.com/manjaro/unstable/$repo/$arch
::Germany         : https://manjaro.moson.eu/unstable/$repo/$arch
::Germany         : https://mirror.philpot.de/manjaro/unstable/$repo/$arch
::Germany         : http://mirror.ragenetwork.de/manjaro/unstable/$repo/$arch
::Germany         : https://mirror.alpix.eu/manjaro/unstable/$repo/$arch
::Germany         : https://mirror.netzspielplatz.de/manjaro/packages/unstable/$repo/$arch
::INFO Mirror list generated and saved to: /etc/pacman.d/mirrorlist
::INFO To reset custom mirrorlist 'sudo pacman-mirrors -id'
::INFO To remove custom config run  'sudo pacman-mirrors -c all'

Next test is same - just using the --test-async argument

~ >>> sudo pacman-mirrors -c Germany --test-async                                                                     
[sudo] password for fh: 
::INFO Downloading mirrors from repo.manjaro.org
::INFO User generated mirror list
::------------------------------------------------------------
::INFO Custom mirror file saved: /var/lib/pacman-mirrors/custom-mirrors.json
::INFO Using custom mirror file
::INFO Querying mirrors - This may take some time
     0.1670 :: Germany         - http://mirror.philpot.de/manjaro/
     0.2280 :: Germany         - https://mirror.alpix.eu/manjaro/
     0.1950 :: Germany         - https://mirror.philpot.de/manjaro/
     0.1340 :: Germany         - http://manjaro.moson.eu/
     0.1410 :: Germany         - http://mirror.23media.com/manjaro/
     0.1830 :: Germany         - https://manjaro.moson.eu/
     0.1970 :: Germany         - http://mirror.ragenetwork.de/manjaro/
     0.2070 :: Germany         - https://mirror.23media.com/manjaro/
     0.2610 :: Germany         - https://mirror.netzspielplatz.de/manjaro/packages/
::INFO Writing mirror list
::Germany         : https://mirror.alpix.eu/manjaro/unstable/$repo/$arch
::Germany         : https://mirror.philpot.de/manjaro/unstable/$repo/$arch
::Germany         : https://manjaro.moson.eu/unstable/$repo/$arch
::Germany         : http://mirror.ragenetwork.de/manjaro/unstable/$repo/$arch
::Germany         : https://mirror.23media.com/manjaro/unstable/$repo/$arch
::Germany         : https://mirror.netzspielplatz.de/manjaro/packages/unstable/$repo/$arch
::INFO Mirror list generated and saved to: /etc/pacman.d/mirrorlist
::INFO To reset custom mirrorlist 'sudo pacman-mirrors -id'
::INFO To remove custom config run  'sudo pacman-mirrors -c all'

Forum kindly sponsored by