Advertisement
Not a member of Pastebin yet?
Sign Up,
it unlocks many cool features!
- package uptobox
- import (
- "github.com/ncw/rclone/fs"
- "github.com/pkg/errors"
- "net/http"
- "net/http/cookiejar"
- "net/url"
- "encoding/json"
- "time"
- "io"
- "github.com/ncw/rclone/dircache"
- "path"
- "strings"
- "github.com/PuerkitoBio/goquery"
- "regexp"
- "strconv"
- "fmt"
- "mime/multipart"
- "bytes"
- "io/ioutil"
- )
- func init() {
- fs.Register(&fs.RegInfo{
- Name: "Uptobox",
- Description: "Uptobox Drive",
- NewFs: NewFs,
- Options: []fs.Option{
- {
- Name: "name",
- Help: "Uptobox username",
- Optional: false,
- },
- {
- Name: "pass",
- Help: "Uptobox password",
- Optional: false,
- IsPassword: true,
- },
- },
- })
- }
- type Fs struct {
- name string
- root string
- features *fs.Features // optional features
- token string
- dirCache *dircache.DirCache
- trueRootID string
- httpClient *http.Client
- }
- type Object struct {
- fs *Fs
- remote string
- id string
- size int64
- date time.Time
- numId string
- }
- type loginResponse struct {
- Error string `json:"error"`
- Success string `json:"success"`
- Msg string `json:"msg"`
- }
- // NewFs constructs an Fs from the path, container:path
- func NewFs(name, root string) (fs.Fs, error) {
- root = strings.Trim(root, "/")
- user := fs.ConfigFileGet(name, "name")
- pass := fs.ConfigFileGet(name, "pass")
- if user == "" {
- return nil, errors.New("No username specified")
- }
- if pass == "" {
- return nil, errors.New("No password speciefied")
- }
- pass, err := fs.Reveal(pass)
- if err != nil {
- return nil, err
- }
- // Get auth cookie
- resp, err := http.PostForm("https://login.uptobox.com/logarithme",
- url.Values{"login": {user}, "password": {pass}, "op": {"login"}})
- if err != nil || resp.StatusCode != 200 {
- return nil, errors.New("Can't login")
- }
- var jsonResponse loginResponse
- err = json.NewDecoder(resp.Body).Decode(&jsonResponse)
- if err != nil {
- return nil, err
- }
- if jsonResponse.Error != "" {
- return nil, errors.New(jsonResponse.Error)
- }
- // Get Cookie
- cookie := ""
- for _, ck := range resp.Cookies() {
- if ck.Name == "xfss" {
- cookie = ck.Value
- break
- }
- }
- if cookie == "" {
- return nil, errors.New("Invalid authentification cookie")
- }
- // Create http client with auth cookie
- u, _ := url.Parse("https://uptobox.com")
- jar, _ := cookiejar.New(nil)
- jar.SetCookies(u, []*http.Cookie{{
- Name: "xfss",
- Value: cookie,
- Path: "/",
- Domain: ".uptobox.com",
- }})
- // FS structure
- f := &Fs{
- name: name,
- root: root,
- token: "",
- trueRootID: "0",
- httpClient: &http.Client{
- Jar: jar,
- CheckRedirect: func(req *http.Request, via []*http.Request) error {
- return http.ErrUseLastResponse
- },
- },
- }
- //Retrieve Uptobox Token
- resp, err = f.httpClient.Get("https://uptobox.com/?op=my_files")
- defer resp.Body.Close()
- doc, err := goquery.NewDocumentFromReader(resp.Body)
- if err != nil {
- return nil, err
- }
- // Extract token
- // Todo : Optimize
- /*tokenInput := doc.Find(".input_append input[name=token]")
- tokenValue := tokenInput.Attr("value")
- if tokenValue == nil {
- return nil, errors.New("Can't retrieve token")
- }
- f.token = tokenValue*/
- doc.Find(".input_append input[name=token]").Each(func(idx int, sel *goquery.Selection) {
- for _, a := range sel.Nodes[0].Attr {
- if a.Key == "value" {
- f.token = a.Val
- }
- }
- })
- // Set features
- f.features = (&fs.Features{}).Fill(f)
- // Init dircache
- f.dirCache = dircache.New(root, "0", f)
- err = f.dirCache.FindRoot(false)
- if err != nil {
- // Assume it is a file
- newRoot, remote := dircache.SplitPath(root)
- newF := *f
- newF.dirCache = dircache.New(newRoot, f.trueRootID, &newF)
- newF.root = newRoot
- // Make new Fs which is the parent
- err = newF.dirCache.FindRoot(false)
- if err != nil {
- // No root so return old f
- return f, nil
- }
- //Find File
- _, err := f.FindFile(newF.dirCache.RootID(), remote)
- if err != nil {
- if err == fs.ErrorObjectNotFound {
- // File doesn't exist so return old f
- return f, nil
- }
- return nil, err
- }
- // return an error with an fs which points to the parent
- return &newF, fs.ErrorIsFile
- }
- // Return fs
- return f, nil
- }
- // Name of the remote (as passed into NewFs)
- func (f *Fs) Name() string {
- return f.name
- }
- // Root of the remote (as passed into NewFs)
- func (f *Fs) Root() string {
- return f.root
- }
- // String converts this Fs to a string
- func (f *Fs) String() string {
- return fmt.Sprintf("Uptobox")
- }
- // Precision return the precision of this Fs
- func (f *Fs) Precision() time.Duration {
- return time.Second
- }
- // Hashes returns the supported hash sets.
- func (f *Fs) Hashes() fs.HashSet {
- return fs.HashSet(fs.HashNone)
- }
- // Features returns the optional features of this Fs
- func (f *Fs) Features() *fs.Features {
- return f.features
- }
- // ListDir reads the directory specified by the job into out, returning any more jobs
- func (f *Fs) ListDir(out fs.ListOpts, job dircache.ListDirJob) (jobs []dircache.ListDirJob, err error) {
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Send request
- resp, err := f.httpClient.Get("https://uptobox.com/?op=my_files&fld_id=" + job.DirID)
- // Check http code and retry
- if err != nil || resp.StatusCode != 200 {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Directory listing failed for %q, low level retry %d/%d", job.Path, tries, maxTries)
- continue
- }
- // Parse HTML
- doc, _ := goquery.NewDocumentFromReader(resp.Body)
- // Extract Files
- doc.Find("table.files .cell_files").Each(func(idx int, sel *goquery.Selection) {
- name, _ := sel.Find("td:nth-child(2) a").Html()
- fileUrl, _ := sel.Find("td:nth-child(2) a").Attr("href")
- id := path.Base(fileUrl)
- // Only for real files
- if name != " " && name != "" && name != " " {
- fullfile, err := f.GetFile(id, job.Path)
- if err != nil {
- return
- }
- out.Add(fullfile)
- }
- })
- // Extract Folders
- doc.Find("table.files td:not([style='max-width:1000px;']):nth-child(1) a").Each(func(idx int, sel *goquery.Selection) {
- reg, _ := regexp.Compile("(.*) \\((.*)$")
- name := reg.ReplaceAllString(sel.Text(), "$1")
- dirlink, _ := sel.Attr("href")
- dirid := strings.Replace(dirlink, "?op=my_files&fld_id=", "", 1)
- if out.IncludeDirectory(job.Path + name) {
- // Only for reals paths
- if name != " " && name != "" && name != " " {
- // Create dir item
- dir := &fs.Dir{
- Name: job.Path + name,
- Bytes: -1,
- Count: -1,
- }
- // Add directory
- if out.AddDir(dir) {
- return
- }
- // Recursive path
- if job.Depth > 0 {
- jobs = append(jobs, dircache.ListDirJob{DirID: dirid, Path: job.Path + name + "/", Depth: job.Depth - 1})
- }
- }
- }
- })
- // Close HTML Request
- resp.Body.Close()
- // Return directory
- return jobs, nil
- }
- // Error
- return nil, err
- }
- // List walks the path returning files and directories into out
- func (f *Fs) List(out fs.ListOpts, dir string) {
- f.dirCache.List(f, out, dir)
- }
- func (f *Fs) NewObject(remote string) (fs.Object, error) {
- return f.FindFile(f.dirCache.RootID(), remote)
- }
- func (f *Fs) Put(in io.Reader, src fs.ObjectInfo) (fs.Object, error) {
- f.dirCache.FindRoot(true)
- resp, err := f.httpClient.Get("https://uptobox.com/")
- defer resp.Body.Close()
- doc, _ := goquery.NewDocumentFromReader(resp.Body)
- uploadUrl, _ := doc.Find("form#fileupload").Attr("action")
- var bodyReader io.Reader
- bodyReader, bodyWriter := io.Pipe()
- writer := multipart.NewWriter(bodyWriter)
- contentType := writer.FormDataContentType()
- contentLength := int64(-1)
- buf := make([]byte, 1)
- n, err := io.ReadFull(in, buf)
- isZeroLength := err == io.EOF
- if !isZeroLength && err != nil {
- return nil, err
- }
- in = io.MultiReader(bytes.NewReader(buf[:n]), in)
- errChan := make(chan error, 1)
- go func() {
- defer bodyWriter.Close()
- var err error
- part, err := writer.CreateFormFile("files[]", src.Remote())
- if err != nil {
- errChan <- err
- return
- }
- if _, err := io.Copy(part, in); err != nil {
- errChan <- err
- return
- }
- errChan <- writer.Close()
- }()
- if isZeroLength {
- buf, err := ioutil.ReadAll(bodyReader)
- if err != nil {
- return nil, err
- }
- bodyReader = bytes.NewReader(buf)
- contentLength = int64(len(buf))
- }
- req, err := http.NewRequest("POST", uploadUrl, bodyReader)
- if err != nil {
- return nil, err
- }
- req.ContentLength = contentLength
- req.Header.Add("Content-Type", contentType)
- f.httpClient.Do(req)
- return f.FindFile(f.dirCache.RootID(), src.Remote())
- }
- func (f *Fs) Mkdir(dir string) error {
- err := f.dirCache.FindRoot(true)
- if err != nil {
- return err
- }
- if dir != "" {
- _, err = f.dirCache.FindDir(dir, true)
- }
- return err
- }
- func (f *Fs) Rmdir(dir string) error {
- panic("implement Rmdir")
- }
- // Get Files informations from uid
- func (f *Fs) GetFile(fileuid string, basepath string) (*Object, error) {
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Get advanced informations
- fileInfo, err := f.httpClient.Get("https://uptobox.com/" + fileuid)
- // Check http code and retry
- if err != nil || (fileInfo.StatusCode != 200 && fileInfo.StatusCode != 302) {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Advanced file informations error for %q, low level retry %d/%d", fileuid, tries, maxTries)
- continue
- }
- // Parse HTML
- fileHTML, _ := goquery.NewDocumentFromReader(fileInfo.Body)
- fileSize, _ := fileHTML.Find("input[name='file_size_real']").Attr("value")
- fullName, _ := fileHTML.Find("input[name='fname']").Attr("value")
- rsize, _ := strconv.ParseInt(fileSize, 10, 64)
- // Close HTTP
- fileInfo.Body.Close()
- // Return Object
- return &Object{
- fs: f,
- remote: path.Join(basepath, fullName),
- id: fileuid,
- size: rsize,
- date: time.Now(),
- }, nil
- // TODO : Get Upload time from HEAD request on file download link
- // TODO : Cache File request and File HEAD requests
- }
- return nil, fs.ErrorObjectNotFound
- }
- func (f *Fs) FindFile(pathID string, filename string) (*Object, error) {
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Send request
- resp, err := f.httpClient.Get("https://uptobox.com/?op=my_files&fld_id=" + pathID)
- // Check http code and retry
- if err != nil || resp.StatusCode != 200 {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Directory listing failed for id %q, low level retry %d/%d", pathID, tries, maxTries)
- continue
- }
- doc, _ := goquery.NewDocumentFromReader(resp.Body)
- var obj *Object = nil
- doc.Find("table.files .cell_files").Each(func(idx int, sel *goquery.Selection) {
- fileUrl, _ := sel.Find("td:nth-child(2) a").Attr("href")
- id := path.Base(fileUrl)
- fullfile, err := f.GetFile(id, "")
- fullfile.numId, _ = sel.Find("td:nth-child(1) input").Attr("value")
- if err == nil && path.Base(fullfile.remote) == filename {
- obj = fullfile
- }
- })
- if obj == nil {
- return nil, fs.ErrorObjectNotFound
- }
- return obj, nil
- }
- return nil, errors.New("Failed to resolve uptobox")
- }
- // FindLeaf finds a directory of name leaf in the folder with ID pathID
- func (f *Fs) FindLeaf(pathID, leaf string) (pathIDOut string, found bool, err error) {
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Send request
- resp, err := f.httpClient.Get("https://uptobox.com/?op=my_files&fld_id=" + pathID)
- // Check http code and retry
- if err != nil || resp.StatusCode != 200 {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Directory listing failed for id %q, low level retry %d/%d", pathID, tries, maxTries)
- continue
- }
- // Parse HTML
- doc, _ := goquery.NewDocumentFromReader(resp.Body)
- // Out ID
- outID := ""
- // Extract Folders
- doc.Find("table.files td:not([style='max-width:1000px;']):nth-child(1) a").Each(func(idx int, sel *goquery.Selection) {
- reg, _ := regexp.Compile("(.*) \\((.*)$")
- name := reg.ReplaceAllString(sel.Text(), "$1")
- dirlink, _ := sel.Attr("href")
- dirid := strings.Replace(dirlink, "?op=my_files&fld_id=", "", 1)
- if name == leaf {
- outID = dirid
- }
- })
- // Close HTML Request
- resp.Body.Close()
- // Return directory
- if outID == "" {
- return "", false, nil
- }
- // Return id
- return outID, true, nil
- }
- // Error
- return "", false, nil
- }
- // CreateDir makes a directory with pathID as parent and name leaf
- func (f *Fs) CreateDir(pathID, leaf string) (newID string, err error) {
- resp, err := f.httpClient.PostForm("https://uptobox.com/", url.Values{
- "op": {"my_files"},
- "create_new_folder": {leaf},
- "fld_id": {pathID},
- "key": {""},
- "pub": {"1"},
- "to_folder": {"0"},
- "token": {f.token},
- })
- if err != nil {
- return "", err
- }
- if resp.StatusCode != 302 {
- return "", errors.New("Could not create dir")
- }
- childID, found, err := f.FindLeaf(pathID, leaf)
- if found != true {
- return "", errors.New("Failed to create directory")
- }
- return childID, err
- }
- // Return a string version
- func (o *Object) String() string {
- if o == nil {
- return "<nil>"
- }
- return o.remote
- }
- // Remote returns the remote path
- func (o *Object) Remote() string {
- return o.remote
- }
- // ModTime returns the modification time of the object
- func (o *Object) ModTime() time.Time {
- /*
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Time
- filetime := ""
- for tries := 1; tries <= maxTries; tries++ {
- // Send request
- resp, err := o.fs.httpClient.Head("http://www59.uptobox.com/d/tqy2g7wsh7dnpxkqdkhjgwj3moaakjwgztax7ex2cwfjviss2vubxn6l7g6caxyq42oyrbhm6qo6c2snk4/Bricks%20in%20Motion.mkv")
- // Check http code and retry
- if err != nil || resp.StatusCode != 200 {
- time.Sleep(time.Duration(tries * 500) * time.Millisecond)
- fs.Debugf(o.fs, "Get modtime failed for id %q, low level retry %d/%d", o.id, tries, maxTries)
- continue
- }
- // Set time
- filetime = resp.Header.Get("last-date-modified")
- // Close HTTP request
- resp.Body.Close()
- // Return
- break;
- }*/
- return time.Now()
- }
- // Size returns the size of an object in bytes
- func (o *Object) Size() int64 {
- return o.size
- }
- func (o *Object) Fs() fs.Info {
- return o.fs
- }
- // Hash returns nothing, not supported
- func (o *Object) Hash(fs.HashType) (string, error) {
- return "", nil
- }
- // Storable returns a boolean showing whether this object storable
- func (o *Object) Storable() bool {
- return true
- }
- // SetModTime return an error, not supported
- func (o *Object) SetModTime(time.Time) error {
- return fs.ErrorCantSetModTime
- }
- // Get download URL from UID
- func (f *Fs) GetDownloadURL(fileuid string) (string, error) {
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Get advanced informations
- fileInfo, err := f.httpClient.Get("https://uptobox.com/" + fileuid)
- // Check http code and retry
- if err != nil || (fileInfo.StatusCode != 200 && fileInfo.StatusCode != 302) {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Download-link generation failed (%q), low level retry %d/%d", fileuid, tries, maxTries)
- continue
- }
- // Get 302 redirect
- if fileInfo.StatusCode == 302 {
- return string(fileInfo.Header.Get("Location")), nil
- }
- // Parse values
- fileHTML, _ := goquery.NewDocumentFromReader(fileInfo.Body)
- fileSize, _ := fileHTML.Find("input[name='file_size_real']").Attr("value")
- fullName, _ := fileHTML.Find("input[name='fname']").Attr("value")
- rand, _ := fileHTML.Find("input[name='rand']").Attr("value")
- // Close HTTP
- fileInfo.Body.Close()
- // If Auto 302 is disabled, execute second request
- fileDownload, err := f.httpClient.PostForm("https://uptobox.com/"+fileuid, url.Values{
- "op": {"download2"},
- "id": {fileuid},
- "fname": {fullName},
- "file_size_real": {fileSize},
- "rand": {rand},
- "referer": {"https://uptobox.com/?op=my_files"},
- "method_free": {""},
- "method_premium": {""},
- "down_direct": {"1"},
- })
- // Check http code and retry
- if err != nil || (fileInfo.StatusCode != 200) {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Download-link generation failed (%q), low level retry %d/%d", fileuid, tries, maxTries)
- continue
- }
- // Extract link
- fileHTML, _ = goquery.NewDocumentFromReader(fileDownload.Body)
- url, _ := fileHTML.Find(".bg_page div div:nth-child(2) a").Attr("href")
- // Close HTTP
- fileDownload.Body.Close()
- // Return url
- return url, nil
- }
- // Return error
- return "", errors.New("Download-link generation failed")
- }
- // Open an object for read
- func (o *Object) Open(options ...fs.OpenOption) (in io.ReadCloser, err error) {
- fileUrl, err := o.fs.GetDownloadURL(o.id)
- if err != nil {
- return nil, err
- }
- resp, err := o.fs.httpClient.Get(fileUrl)
- return resp.Body, err
- }
- func (o *Object) Update(in io.Reader, src fs.ObjectInfo) error {
- panic("implement Update")
- }
- func (o *Object) Remove() error {
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- // Send request
- resp, _ := o.fs.httpClient.Get("https://uptobox.com/?op=my_files&del_code=" + o.id + "&token=" + o.fs.token)
- // Check success
- if resp.StatusCode == 302 || resp.StatusCode == 200 {
- return nil
- }
- }
- return errors.New("File deletion failed")
- }
- func (f *Fs) Move(src fs.Object, remote string) (fs.Object, error) {
- panic("sqdqsdsqd")
- // Source dir
- _, srcDirectoryID, errSrc := f.dirCache.FindPath(path.Dir(src.Remote()), false)
- // Check source path
- if errSrc != nil {
- return src, errSrc
- }
- // Find remote path
- _, dstDirectoryID, errDst := f.dirCache.FindPath(remote, true)
- // Check path exists
- if errDst != nil {
- return src, errDst
- }
- // Get file
- file, errFile := f.FindFile(srcDirectoryID, path.Base(remote))
- // Check file return
- if errFile != nil {
- return src, errFile
- }
- // Get max tries value
- maxTries := fs.Config.LowLevelRetries
- // Try
- for tries := 1; tries <= maxTries; tries++ {
- println("fld_id : " + srcDirectoryID)
- println("file_id : " + file.numId)
- println("fld_id : " + dstDirectoryID)
- // Send request
- resp, err := f.httpClient.PostForm("https://uptobox.com/", url.Values{
- "create_new_folder": {""},
- "op": {"my_files"},
- "token": {f.token},
- "fld_id": {srcDirectoryID},
- "key": {""},
- "file_id": {file.numId},
- "pub": {"0"},
- "to_folder": {dstDirectoryID},
- "to_folder_move": {"Move files"},
- })
- // Check http code and retry
- if err != nil || (resp.StatusCode != 302 && resp.StatusCode != 200) {
- time.Sleep(time.Duration(tries*500) * time.Millisecond)
- fs.Debugf(f, "Moving file failed (%q), low level retry %d/%d", file.id, tries, maxTries)
- continue
- }
- // Get new file
- newFileObj, errObj := f.GetFile(file.id, remote)
- // Check success
- if errObj == nil {
- return newFileObj, nil
- }
- // Error break
- break
- }
- return nil, errors.New("Moving file failed")
- }
- var (
- _ fs.Fs = &Fs{}
- _ fs.Mover = &Fs{}
- _ fs.Object = &Object{}
- )
Advertisement
Add Comment
Please, Sign In to add comment
Advertisement