Skip to content

Commit

Permalink
Task download, cancel, unzipping
Browse files Browse the repository at this point in the history
  • Loading branch information
pierotofy committed Jan 12, 2019
1 parent 304359d commit 991b39c
Show file tree
Hide file tree
Showing 14 changed files with 567 additions and 23 deletions.
23 changes: 20 additions & 3 deletions internal/cmd/root.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@
package cmd

import (
"fmt"
"mime"
"os"
"path/filepath"
Expand All @@ -33,6 +32,7 @@ import (

var outputPath string
var nodeName string
var force bool

var rootCmd = &cobra.Command{
Use: "odm [flags] <images> [<gcp>] [args]",
Expand All @@ -45,6 +45,15 @@ var rootCmd = &cobra.Command{
os.Exit(0)
}

// Check output directory
filesCount, err := fs.DirectoryFilesCount(outputPath)
if err != nil {
logger.Error(err)
}
if filesCount > 0 && !force {
logger.Error(outputPath + " already exists (pass --force to override directory contents)")
}

inputFiles, options := parseArgs(args)
inputFiles = filterImagesAndText(inputFiles)

Expand Down Expand Up @@ -74,7 +83,15 @@ var rootCmd = &cobra.Command{
logger.Error(err)
}

odm.Run(inputFiles, parseOptions(options, nodeOptions), *node)
// Create output directory
if !fs.IsDirectory(outputPath) {
err = os.MkdirAll(outputPath, 0755)
if err != nil {
logger.Error(err)
}
}

odm.Run(inputFiles, parseOptions(options, nodeOptions), *node, outputPath)
},

TraverseChildren: true,
Expand All @@ -95,6 +112,7 @@ func init() {
rootCmd.PersistentFlags().BoolVarP(&logger.DebugFlag, "debug", "d", false, "show debug output")
rootCmd.PersistentFlags().BoolVarP(&logger.QuietFlag, "quiet", "q", false, "suppress output")

rootCmd.Flags().BoolVarP(&force, "force", "f", false, "replace the contents of the output directory if it already exists")
rootCmd.Flags().StringVarP(&outputPath, "output", "o", "./output", "directory where to store processing results")
rootCmd.Flags().StringVarP(&nodeName, "node", "n", "default", "Processing node to use")
rootCmd.Flags().SetInterspersed(false)
Expand All @@ -118,7 +136,6 @@ func parseArgs(args []string) ([]string, []string) {
}
}
} else if fs.IsFile(arg) {
fmt.Printf(arg)
inputFiles = append(inputFiles, arg)
} else {
options = append(options, arg)
Expand Down
15 changes: 15 additions & 0 deletions internal/config/auth.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package config

import (
Expand Down
17 changes: 15 additions & 2 deletions internal/config/configuration.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package config

import (
Expand Down Expand Up @@ -38,8 +53,6 @@ func (c Configuration) Save() {
type Configuration struct {
Nodes map[string]odm.Node `json:"nodes"`

Tasks []odm.Task `json:"tasks"`

filePath string
}

Expand Down
15 changes: 15 additions & 0 deletions internal/config/configuration_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package config

import (
Expand Down
15 changes: 15 additions & 0 deletions internal/config/publicnodes.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package config

import (
Expand Down
39 changes: 38 additions & 1 deletion internal/fs/fs.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,24 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package fs

import "os"
import (
"io/ioutil"
"os"
)

// FileExists checks if a file path exists
func FileExists(filePath string) (bool, error) {
Expand Down Expand Up @@ -33,3 +51,22 @@ func IsFile(path string) bool {
}
return !fileInfo.IsDir()
}

// DirectoryFilesCount returns the number of files in a directory. If the dir
// does not exists, it returns 0.
func DirectoryFilesCount(dirPath string) (int, error) {
if _, err := os.Stat(dirPath); err != nil {
if os.IsNotExist(err) {
return 0, nil
} else {
return -1, err
}
}

files, err := ioutil.ReadDir(dirPath)
if err != nil {
return -1, err
}

return len(files), nil
}
106 changes: 106 additions & 0 deletions internal/fs/unzip.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,106 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

// Based on work published on https://golangcode.com/download-a-file-with-progress/

package fs

import (
"archive/zip"
"fmt"
"io"
"os"
"path/filepath"
"strings"
"time"

"github.com/OpenDroneMap/CloudODM/internal/logger"
"github.com/cheggaaa/pb"
)

// Unzip decompresses a zip archive
func Unzip(zipFilePath string, destDirectory string) ([]string, error) {
var bar *pb.ProgressBar
var filenames []string

r, err := zip.OpenReader(zipFilePath)
if err != nil {
return filenames, err
}
defer r.Close()

showProgress := !logger.QuietFlag && len(r.File) > 0
if showProgress {
bar = pb.New(len(r.File)).SetUnits(pb.U_NO).SetRefreshRate(time.Millisecond * 10)
bar.Start()
defer bar.Finish()
}

for _, f := range r.File {

rc, err := f.Open()
if err != nil {
return filenames, err
}
defer rc.Close()

if showProgress {
bar.Prefix("[" + f.Name + "]")
}

// Store filename/path for returning and using later on
fpath := filepath.Join(destDirectory, f.Name)

// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
if !strings.HasPrefix(fpath, filepath.Clean(destDirectory)+string(os.PathSeparator)) {
return filenames, fmt.Errorf("%s: illegal file path", fpath)
}

filenames = append(filenames, fpath)

if f.FileInfo().IsDir() {

// Make Folder
os.MkdirAll(fpath, os.ModePerm)

} else {

// Make File
if err = os.MkdirAll(filepath.Dir(fpath), os.ModePerm); err != nil {
return filenames, err
}

outFile, err := os.OpenFile(fpath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
if err != nil {
return filenames, err
}

_, err = io.Copy(outFile, rc)

// Close the file without defer to close before next iteration of loop
outFile.Close()

if err != nil {
return filenames, err
}
}

if showProgress {
bar.Increment()
}
}

return filenames, nil
}
15 changes: 15 additions & 0 deletions internal/io/terminal.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,18 @@
// Copyright © 2018 CloudODM Contributors
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package io

import (
Expand Down
Loading

0 comments on commit 991b39c

Please sign in to comment.