初步完成
15128022404
2 years ago
parent
4a0f9516a3
commit
b7b913bc74
@ -0,0 +1,37 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/forget-the-bright/grvm/internal/pkg/config"
|
||||
"github.com/forget-the-bright/grvm/internal/pkg/proxy"
|
||||
)
|
||||
|
||||
func getGradleAllInfo() []*GradleItem {
|
||||
resp, _ := proxy.HttpGetByProxy(Collector_Archive_Url)
|
||||
resp_checksums, _ := proxy.HttpGetByProxy(Collector_Release_Checksums)
|
||||
|
||||
defer resp.Body.Close()
|
||||
defer resp_checksums.Body.Close()
|
||||
doc_selector, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
doc__checksums_selector, _ := goquery.NewDocumentFromReader(resp_checksums.Body)
|
||||
|
||||
grvms := make([]*GradleItem, 0)
|
||||
//获取所有的sha256
|
||||
lis := doc__checksums_selector.Find(".layout__main").Find("ul[style]")
|
||||
|
||||
resources_docs := doc_selector.Find(".resources-contents")
|
||||
//获取所有的版本号
|
||||
a_docs := resources_docs.Find("a[name]")
|
||||
//获取所有的发布时间
|
||||
version_times := resources_docs.Find("p[class='u-text-with-icon u-no-margin-bottom u-no-margin-top']")
|
||||
|
||||
a_docs.Each(func(j int, a_doc *goquery.Selection) {
|
||||
version := a_doc.AttrOr("name", "")
|
||||
sha256 := lis.Eq(j).Find("li").Eq(0).Find("code").Text()
|
||||
version_time := version_times.Eq(j).Find("span").Eq(1).Text()
|
||||
//fmt.Printf("version: %v time: %v sha256: %v\n", version, "", sha256)
|
||||
grvms = append(grvms, build_GradleItem(version, version_time, sha256))
|
||||
})
|
||||
//fmt.Printf("len(grvms): %v\n", len(grvms))
|
||||
return config.ReverseArray(grvms)
|
||||
}
|
||||
@ -1,215 +0,0 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
"runtime"
|
||||
"strings"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
"github.com/forget-the-bright/j/internal/pkg/config"
|
||||
)
|
||||
|
||||
func ConvertCollectorToUrlItem(colls []*Collector, isGetSha256 bool) []*config.UrlItem {
|
||||
var rs = make([]*config.UrlItem, 0)
|
||||
for _, coll := range colls {
|
||||
var item *Op_Item
|
||||
switch runtime.GOOS {
|
||||
case "linux":
|
||||
if runtime.GOARCH == "aarch64" {
|
||||
item = coll.Linux_AArch64
|
||||
} else {
|
||||
item = coll.Linux_X64
|
||||
}
|
||||
case "windows":
|
||||
item = coll.Windows_X64
|
||||
case "darwin":
|
||||
if runtime.GOARCH == "aarch64" {
|
||||
item = coll.Mac_AArch64
|
||||
} else {
|
||||
item = coll.Mac_X64
|
||||
}
|
||||
default:
|
||||
item = nil
|
||||
}
|
||||
if item != nil {
|
||||
rs = append(rs, &config.UrlItem{
|
||||
In: &config.JavaFileItem{
|
||||
FileName: item.FileName,
|
||||
URL: item.Url,
|
||||
Sha256: GetSha256ByUrl(item.Sha256Url, isGetSha256),
|
||||
},
|
||||
SimpleName: coll.Version,
|
||||
Expected: getFileNameNoSuffix(item.FileName),
|
||||
})
|
||||
}
|
||||
}
|
||||
switch runtime.GOOS {
|
||||
case "windows":
|
||||
rs = append(rs, &config.UrlItem{
|
||||
In: &config.JavaFileItem{
|
||||
FileName: "openjdk-8u42-b03-windows-i586-14_jul_2022.zip",
|
||||
URL: "https://download.java.net/openjdk/jdk8u42/ri/openjdk-8u42-b03-windows-i586-14_jul_2022.zip",
|
||||
Sha256: "0314134bd981db63c7ca68d262ef896383b5694307a14bac81af88b5ad926279",
|
||||
},
|
||||
Expected: "openjdk-8u42-b03-windows-i586-14_jul_2022",
|
||||
SimpleName: "8",
|
||||
})
|
||||
case "linux":
|
||||
rs = append(rs, &config.UrlItem{
|
||||
In: &config.JavaFileItem{
|
||||
FileName: "openjdk-8u42-b03-linux-x64-14_jul_2022.tar.gz",
|
||||
URL: "https://download.java.net/openjdk/jdk8u42/ri/openjdk-8u42-b03-linux-x64-14_jul_2022.tar.gz",
|
||||
Sha256: "dd5fc6ef5ebffb88cd66af5258226c31f6c719fdcd855d95464fdb2cab051baa",
|
||||
},
|
||||
Expected: "openjdk-8u42-b03-linux-x64-14_jul_2022",
|
||||
SimpleName: "8",
|
||||
})
|
||||
|
||||
}
|
||||
return config.ReverseArray(rs)
|
||||
}
|
||||
|
||||
func GetOpenJDKVesionUrlInfo() []*Collector {
|
||||
resp, _ := http.Get(Collector_Archive_Url)
|
||||
Collectors = make([]*Collector, 0)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
doc_selector, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
divs := doc_selector.Find("#sidebar").Find(".links")
|
||||
divs.Each(func(j int, div *goquery.Selection) {
|
||||
link_about := div.Find(".about").Text()
|
||||
if link_about == "Reference Implementations" {
|
||||
a_docs := div.Find("a")
|
||||
a_docs.Each(func(j int, a_doc *goquery.Selection) {
|
||||
info_url := a_doc.AttrOr("href", "")
|
||||
versions := strings.Split(a_doc.Text(), " ")
|
||||
version := versions[len(versions)-1]
|
||||
version_url := Collector_Url + strings.ReplaceAll(info_url, ".", "")
|
||||
Collectors = append(Collectors, getVersionByUrl(version, version_url))
|
||||
//fmt.Println(version)
|
||||
//fmt.Println(version_url)
|
||||
})
|
||||
}
|
||||
//fmt.Printf("link_about: %v\n", link_about)
|
||||
})
|
||||
return Collectors
|
||||
}
|
||||
|
||||
func GetOpenJDKArchiveReleasesInfo() []*Collector {
|
||||
Archive_Releases_Collectors = make([]*Collector, 0)
|
||||
resp, _ := http.Get(Collector_Archive_Url)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
docs, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
tbody_docs := docs.Find(".builds").Find("tbody").Children()
|
||||
var collector_Item *Collector
|
||||
tbody_docs.Each(func(j int, tr *goquery.Selection) {
|
||||
th := tr.Find("th")
|
||||
if th.Length() == 1 {
|
||||
val := th.Text()
|
||||
if val != "Source" {
|
||||
s2 := strings.Split(val, "(build ")
|
||||
version_str := strings.Trim(strings.Trim(s2[0], " "), " GA")
|
||||
//fmt.Println("\n" + version_str)
|
||||
collector_Item = &Collector{
|
||||
Version: version_str,
|
||||
}
|
||||
Archive_Releases_Collectors = append(Archive_Releases_Collectors, collector_Item)
|
||||
}
|
||||
}
|
||||
if th.Length() == 2 {
|
||||
a_item := tr.Find("td").Find("a")
|
||||
file_type := strings.Trim(a_item.First().Text(), "\n")
|
||||
download_url := a_item.First().AttrOr("href", "")
|
||||
file_name := getFileNameByDownLoadUrl(download_url)
|
||||
sha256_url := a_item.Last().AttrOr("href", "")
|
||||
releases := strings.Split(th.First().Text(), "/")
|
||||
|
||||
switch releases[0] {
|
||||
case "Windows":
|
||||
collector_Item.Windows_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
case "Mac":
|
||||
if len(releases) == 1 || releases[1] == "x64" {
|
||||
collector_Item.Mac_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
} else {
|
||||
collector_Item.Mac_AArch64 = build_Op_Item(file_type, releases[1], download_url, sha256_url, file_name)
|
||||
}
|
||||
case "Linux":
|
||||
if len(releases) == 1 || releases[1] == "x64" {
|
||||
collector_Item.Linux_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
} else {
|
||||
collector_Item.Linux_X64 = build_Op_Item(file_type, releases[1], download_url, sha256_url, file_name)
|
||||
}
|
||||
}
|
||||
}
|
||||
})
|
||||
return Archive_Releases_Collectors
|
||||
}
|
||||
|
||||
func getVersionByUrl(version, url string) *Collector {
|
||||
var coll = Collector{
|
||||
Version: version,
|
||||
}
|
||||
resp, _ := http.Get(url)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
doc_selector, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
|
||||
divs := doc_selector.Find("#main").Find("ul")
|
||||
|
||||
if divs.Length() <= 1 || version == "8" {
|
||||
li_docs := divs.Find("li")
|
||||
linux_x64_a := li_docs.Eq(0).Find("a")
|
||||
windows_x64_a := li_docs.Eq(1).Find("a")
|
||||
|
||||
linux_url := linux_x64_a.Eq(0).AttrOr("href", "")
|
||||
linux_file_name := getFileNameByDownLoadUrl(linux_url)
|
||||
linux_file_type := getFileTypeByFileName(linux_file_name)
|
||||
linux_sha256_url := linux_x64_a.Eq(1).AttrOr("href", "")
|
||||
|
||||
windows_url := windows_x64_a.Eq(0).AttrOr("href", "")
|
||||
windows_file_name := getFileNameByDownLoadUrl(windows_url)
|
||||
windows_file_type := getFileTypeByFileName(windows_file_name)
|
||||
windows_sha256_url := windows_x64_a.Eq(1).AttrOr("href", "")
|
||||
|
||||
coll.Linux_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: linux_url,
|
||||
Sha256Url: linux_sha256_url,
|
||||
FileName: linux_file_name,
|
||||
FileType: linux_file_type,
|
||||
}
|
||||
coll.Windows_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: windows_url,
|
||||
Sha256Url: windows_sha256_url,
|
||||
FileName: windows_file_name,
|
||||
FileType: windows_file_type,
|
||||
}
|
||||
} else {
|
||||
divs_eq0 := divs.Eq(0)
|
||||
li_docs := divs_eq0.Find("li")
|
||||
linux_x64_a := li_docs.Eq(0).Find("a")
|
||||
|
||||
linux_url := linux_x64_a.Eq(0).AttrOr("href", "")
|
||||
linux_file_name := getFileNameByDownLoadUrl(linux_url)
|
||||
linux_file_type := getFileTypeByFileName(linux_file_name)
|
||||
linux_sha256_url := linux_x64_a.Eq(1).AttrOr("href", "")
|
||||
coll.Linux_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: linux_url,
|
||||
Sha256Url: linux_sha256_url,
|
||||
FileName: linux_file_name,
|
||||
FileType: linux_file_type,
|
||||
}
|
||||
|
||||
}
|
||||
return &coll
|
||||
}
|
||||
@ -1,188 +0,0 @@
|
||||
package collector
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
func Test_GetSha256ByUrl(t *testing.T) {
|
||||
resp, _ := http.Get("https://download.java.net/java/GA/jdk19.0.1/afdd2e245b014143b62ccb916125e3ce/10/GPL/openjdk-19.0.1_windows-x64_bin.zip.sha256")
|
||||
defer resp.Body.Close()
|
||||
bytes, _ := ioutil.ReadAll(resp.Body)
|
||||
t.Run("", func(t *testing.T) {
|
||||
fmt.Println(string(bytes))
|
||||
})
|
||||
|
||||
}
|
||||
|
||||
func Test_getOpenJdkVersion(t *testing.T) {
|
||||
resp, _ := http.Get(Collector_Archive_Url)
|
||||
colls := make([]*Collector, 0)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
doc_selector, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
t.Run("", func(t *testing.T) {
|
||||
divs := doc_selector.Find("#sidebar").Find(".links")
|
||||
divs.Each(func(j int, div *goquery.Selection) {
|
||||
link_about := div.Find(".about").Text()
|
||||
if link_about == "Reference Implementations" {
|
||||
a_docs := div.Find("a")
|
||||
a_docs.Each(func(j int, a_doc *goquery.Selection) {
|
||||
info_url := a_doc.AttrOr("href", "")
|
||||
versions := strings.Split(a_doc.Text(), " ")
|
||||
version := versions[len(versions)-1]
|
||||
version_url := Collector_Url + strings.ReplaceAll(info_url, ".", "")
|
||||
colls = append(colls, test_getVersionByUrl(version, version_url))
|
||||
//fmt.Println(version)
|
||||
//fmt.Println(version_url)
|
||||
})
|
||||
}
|
||||
fmt.Printf("link_about: %v\n", link_about)
|
||||
})
|
||||
temp := colls
|
||||
fmt.Printf("len(temp): %v\n", len(temp))
|
||||
//fmt.Printf("divs.Length(): %v\n", divs.Length())
|
||||
})
|
||||
}
|
||||
|
||||
func test_getVersionByUrl(version, url string) *Collector {
|
||||
var coll = Collector{
|
||||
Version: version,
|
||||
}
|
||||
resp, _ := http.Get(url)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
doc_selector, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
|
||||
divs := doc_selector.Find("#main").Find("ul")
|
||||
|
||||
if divs.Length() <= 1 || version == "8" {
|
||||
li_docs := divs.Find("li")
|
||||
linux_x64_a := li_docs.Eq(0).Find("a")
|
||||
windows_x64_a := li_docs.Eq(1).Find("a")
|
||||
|
||||
linux_url := linux_x64_a.Eq(0).AttrOr("href", "")
|
||||
linux_file_name := getFileNameByDownLoadUrl(linux_url)
|
||||
linux_file_type := getFileTypeByFileName(linux_file_name)
|
||||
linux_sha256_url := linux_x64_a.Eq(1).AttrOr("href", "")
|
||||
|
||||
windows_url := windows_x64_a.Eq(0).AttrOr("href", "")
|
||||
windows_file_name := getFileNameByDownLoadUrl(windows_url)
|
||||
windows_file_type := getFileTypeByFileName(windows_file_name)
|
||||
windows_sha256_url := windows_x64_a.Eq(1).AttrOr("href", "")
|
||||
|
||||
coll.Linux_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: linux_url,
|
||||
Sha256Url: linux_sha256_url,
|
||||
FileName: linux_file_name,
|
||||
FileType: linux_file_type,
|
||||
}
|
||||
coll.Windows_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: windows_url,
|
||||
Sha256Url: windows_sha256_url,
|
||||
FileName: windows_file_name,
|
||||
FileType: windows_file_type,
|
||||
}
|
||||
} else {
|
||||
divs_eq0 := divs.Eq(0)
|
||||
li_docs := divs_eq0.Find("li")
|
||||
linux_x64_a := li_docs.Eq(0).Find("a")
|
||||
|
||||
linux_url := linux_x64_a.Eq(0).AttrOr("href", "")
|
||||
linux_file_name := getFileNameByDownLoadUrl(linux_url)
|
||||
linux_file_type := getFileTypeByFileName(linux_file_name)
|
||||
linux_sha256_url := linux_x64_a.Eq(1).AttrOr("href", "")
|
||||
coll.Linux_X64 = &Op_Item{
|
||||
Arch: "x64",
|
||||
Url: linux_url,
|
||||
Sha256Url: linux_sha256_url,
|
||||
FileName: linux_file_name,
|
||||
FileType: linux_file_type,
|
||||
}
|
||||
|
||||
}
|
||||
return &coll
|
||||
}
|
||||
|
||||
func Test_getArchiveVersion(t *testing.T) {
|
||||
Archive_Releases_Collectors = make([]*Collector, 0)
|
||||
resp, _ := http.Get(Collector_Archive_Url)
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
fmt.Println("false")
|
||||
}
|
||||
c, _ := goquery.NewDocumentFromReader(resp.Body)
|
||||
t.Run("", func(t *testing.T) {
|
||||
s := c.Find(".builds").Find("tbody").Children()
|
||||
/* s.Each(func(j int, tr *goquery.Selection) {
|
||||
th := tr.Find("th")
|
||||
if th.Length() == 1 {
|
||||
val := th.Text()
|
||||
if val != "Source" {
|
||||
s2 := strings.Split(val, "(build ")
|
||||
fmt.Println(strings.Trim(strings.Trim(s2[0], " "), " GA"))
|
||||
}
|
||||
}
|
||||
}) */
|
||||
var collector_Item *Collector
|
||||
//flag := false
|
||||
s.Each(func(j int, tr *goquery.Selection) {
|
||||
th := tr.Find("th")
|
||||
if th.Length() == 1 {
|
||||
val := th.Text()
|
||||
if val != "Source" {
|
||||
s2 := strings.Split(val, "(build ")
|
||||
version_str := strings.Trim(strings.Trim(s2[0], " "), " GA")
|
||||
fmt.Println("\n" + version_str)
|
||||
collector_Item = &Collector{
|
||||
Version: version_str,
|
||||
}
|
||||
Archive_Releases_Collectors = append(Archive_Releases_Collectors, collector_Item)
|
||||
}
|
||||
}
|
||||
if th.Length() == 2 {
|
||||
a_item := tr.Find("td").Find("a")
|
||||
file_type := strings.Trim(a_item.First().Text(), "\n")
|
||||
download_url := a_item.First().AttrOr("href", "")
|
||||
file_name := getFileNameByDownLoadUrl(download_url)
|
||||
sha256_url := a_item.Last().AttrOr("href", "")
|
||||
releases := strings.Split(th.First().Text(), "/")
|
||||
|
||||
switch releases[0] {
|
||||
case "Windows":
|
||||
collector_Item.Mac_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
case "Mac":
|
||||
if len(releases) == 1 || releases[1] == "x64" {
|
||||
collector_Item.Mac_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
} else {
|
||||
collector_Item.Mac_AArch64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
}
|
||||
case "Linux":
|
||||
if len(releases) == 1 || releases[1] == "x64" {
|
||||
collector_Item.Linux_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
} else {
|
||||
collector_Item.Linux_X64 = build_Op_Item(file_type, "x64", download_url, sha256_url, file_name)
|
||||
}
|
||||
}
|
||||
//fmt.Printf("%v %v\n", strings.Trim(a_item.First().Text(), "\n"), a_item.Last().Text())
|
||||
//fmt.Printf("%v %v\n", th.First().Text(), th.Last().Text())
|
||||
//fmt.Printf("%v\n%v\n", a_item.First().AttrOr("href", ""), a_item.Last().AttrOr("href", ""))
|
||||
//fmt.Printf("th.Text(): %v th.len:%v\n", th.Text(), thLen)
|
||||
}
|
||||
})
|
||||
})
|
||||
c2 := Archive_Releases_Collectors
|
||||
fmt.Printf("len(c2): %v\n", len(c2))
|
||||
fmt.Println(c2)
|
||||
}
|
||||
@ -0,0 +1,49 @@
|
||||
package proxy
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func HttpGetByProxy(url string) (*http.Response, error) {
|
||||
if cli := getClientProxy(); cli != nil {
|
||||
request, _ := http.NewRequest("GET", url, nil)
|
||||
return cli.Do(request)
|
||||
} else {
|
||||
return http.Get(url)
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func getClientProxy() *http.Client {
|
||||
if proxy := getEnvProxy(); proxy == "" {
|
||||
return nil
|
||||
} else {
|
||||
proxyUrl, err := url.Parse(proxy)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return &http.Client{Transport: &http.Transport{Proxy: http.ProxyURL(proxyUrl)}}
|
||||
}
|
||||
}
|
||||
|
||||
func getEnvProxy() string {
|
||||
if proxy_url := getEnvAny("HTTP_PROXY", "http_proxy"); proxy_url != "" {
|
||||
if !strings.Contains(proxy_url, "http://") {
|
||||
return "http://" + proxy_url
|
||||
}
|
||||
return proxy_url
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func getEnvAny(names ...string) string {
|
||||
for _, n := range names {
|
||||
if val := os.Getenv(n); val != "" {
|
||||
return val
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Loading…
Reference in New Issue