From 2d2450868571a03c5b9ad05a57ad653b03a4ecc8 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Mon, 11 Mar 2019 15:09:51 +0100 Subject: [PATCH 01/41] [skip ci] WIP --- Rome.cabal | 2 +- src/Lib.hs | 15 +++---- .../S3Config.hs => Network/AWS/Utils.hs} | 39 ++++++++++++++----- 3 files changed, 38 insertions(+), 18 deletions(-) rename src/{Data/S3Config.hs => Network/AWS/Utils.hs} (57%) diff --git a/Rome.cabal b/Rome.cabal index f234c62..fa85ba4 100644 --- a/Rome.cabal +++ b/Rome.cabal @@ -26,7 +26,6 @@ library , Data.Carthage.Common , Data.Carthage.VersionFile , Data.Romefile - , Data.S3Config , Text.Parsec.Utils , Xcode.DWARF , Caches.S3.Probing @@ -36,6 +35,7 @@ library , Caches.Local.Uploading , Caches.Local.Downloading , Caches.Common + , Network.AWS.Utils build-depends: base >= 4.7 && < 5 , amazonka >= 1.6.1 diff --git a/src/Lib.hs b/src/Lib.hs index 330409c..184db4f 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -38,11 +38,12 @@ import Data.Maybe (fromMaybe, maybe) import Data.Monoid ((<>)) import Data.Romefile import qualified Data.Map.Strict as M (empty) -import qualified Data.S3Config as S3Config import qualified Data.Text as T import qualified Network.AWS as AWS import qualified Network.AWS.Data as AWS (fromText) import qualified Network.AWS.S3 as S3 +import qualified Network.AWS.Utils as AWS + import Network.URL import System.Directory import System.Environment @@ -1208,9 +1209,9 @@ getRegionFromFile => FilePath -- ^ The path to the file containing the `AWS.Region` -> String -- ^ The name of the profile to use -> ExceptT String m AWS.Region -getRegionFromFile f profile = fromFile f $ \file -> ExceptT . return $ do - config <- S3Config.parseS3Config file - S3Config.regionOf (T.pack profile) config +getRegionFromFile f profile = fromFile f $ \fileContents -> ExceptT . return $ do + config <- AWS.parseConfigFile fileContents + AWS.regionOf (T.pack profile) config @@ -1243,7 +1244,7 @@ getEndpointFromFile => FilePath -- ^ The path to the file containing the `AWS.Region` -> String -- ^ The name of the profile to use -> ExceptT String m URL -getEndpointFromFile f profile = fromFile f $ \file -> ExceptT . return $ do - config <- S3Config.parseS3Config file - S3Config.endPointOf (T.pack profile) config +getEndpointFromFile f profile = fromFile f $ \fileContents -> ExceptT . return $ do + config <- AWS.parseConfigFile fileContents + AWS.endPointOf (T.pack profile) config diff --git a/src/Data/S3Config.hs b/src/Network/AWS/Utils.hs similarity index 57% rename from src/Data/S3Config.hs rename to src/Network/AWS/Utils.hs index b560a00..5f8280c 100644 --- a/src/Data/S3Config.hs +++ b/src/Network/AWS/Utils.hs @@ -1,10 +1,13 @@ {-# LANGUAGE OverloadedStrings #-} -module Data.S3Config - ( S3Config - , parseS3Config +module Network.AWS.Utils + ( ConfigFile + , CredentialsFile + , parseConfigFile + , parseCredentialsFile , regionOf , endPointOf + , sourceProfileOf ) where -- For now, only very little information needs to be extracted from the S3 @@ -20,18 +23,28 @@ import qualified Network.AWS as AWS import qualified Network.AWS.Data as AWS import Network.URL -newtype S3Config = S3Config { _ini :: Ini } +newtype ConfigFile = ConfigFile { _awsConfigIni :: Ini } +newtype CredentialsFile = CredentialsFile { _awsCredentialsIni :: Ini } -regionOf :: T.Text -> S3Config -> Either String AWS.Region -regionOf profile = parseRegion <=< lookupValue profile "region" . _ini +class FromIni a where + asIni :: a -> Ini + +instance FromIni ConfigFile where + asIni = _awsConfigIni + +instance FromIni CredentialsFile where + asIni = _awsCredentialsIni + +regionOf :: T.Text -> ConfigFile -> Either String AWS.Region +regionOf profile = parseRegion <=< lookupValue profile "region" . asIni where parseRegion s = if T.null s -- better error message then Left "Failed reading: Failure parsing Region from empty string" else AWS.fromText s -endPointOf :: T.Text -> S3Config -> Either String URL -endPointOf profile = parseURL <=< lookupValue profile "endpoint" . _ini +endPointOf :: T.Text -> ConfigFile -> Either String URL +endPointOf profile = parseURL <=< lookupValue profile "endpoint" . asIni where parseURL s = if T.null s then Left "Failed reading: Failure parsing Endpoint from empty string" @@ -41,5 +54,11 @@ endPointOf profile = parseURL <=< lookupValue profile "endpoint" . _ini . T.unpack $ s -parseS3Config :: T.Text -> Either String S3Config -parseS3Config = fmap S3Config . parseIni +sourceProfileOf :: T.Text -> CredentialsFile -> Either String CredentialsFile +sourceProfileOf p credentialsFile = undefined + +parseConfigFile :: T.Text -> Either String ConfigFile +parseConfigFile = fmap ConfigFile . parseIni + +parseCredentialsFile :: T.Text -> Either String CredentialsFile +parseCredentialsFile = fmap CredentialsFile . parseIni From 5c177d4c9e0e94f864306b3667636c6b9b001b75 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Thu, 4 Apr 2019 17:29:23 +0200 Subject: [PATCH 02/41] [skip ci][WIP] working version with old behavior --- Rome.cabal | 2 +- Rome.podspec | 2 +- app/Main.hs | 2 +- src/Configuration.hs | 8 +++-- src/Lib.hs | 67 ++++++++++++++++++++++++++++++++++++---- src/Network/AWS/Utils.hs | 37 +++++++++++++++++++--- 6 files changed, 103 insertions(+), 15 deletions(-) diff --git a/Rome.cabal b/Rome.cabal index 884dfdf..4e752e0 100644 --- a/Rome.cabal +++ b/Rome.cabal @@ -1,5 +1,5 @@ name: Rome -version: 0.20.0.56 +version: 0.21.0.57 synopsis: A cache for Carthage description: Please see README.md homepage: https://github.com/blender/Rome diff --git a/Rome.podspec b/Rome.podspec index fe51d40..7efd893 100644 --- a/Rome.podspec +++ b/Rome.podspec @@ -1,6 +1,6 @@ Pod::Spec.new do |s| s.name = 'Rome' - s.version = '0.20.0.56' + s.version = '0.21.0.57' s.summary = 'A cache tool for Carthage' s.homepage = 'https://github.com/blender/Rome' s.source = { :http => "#{s.homepage}/releases/download/v#{s.version}/rome.zip" } diff --git a/app/Main.hs b/app/Main.hs index f2fd898..fe86265 100644 --- a/app/Main.hs +++ b/app/Main.hs @@ -10,7 +10,7 @@ import System.Exit romeVersion :: RomeVersion -romeVersion = (0, 20, 0, 56) +romeVersion = (0, 21, 0, 57) diff --git a/src/Configuration.hs b/src/Configuration.hs index 2ef7ef3..d200d07 100644 --- a/src/Configuration.hs +++ b/src/Configuration.hs @@ -33,10 +33,14 @@ getRomefileEntries absoluteRomefilePath = in withExceptT toErr $ fromYaml <|> fromIni where toErr e = "Error while parsing " <> absoluteRomefilePath <> ": " <> e -getS3ConfigFile :: MonadIO m => m FilePath -getS3ConfigFile = ( awsConfigFilePath) `liftM` liftIO getHomeDirectory +getAWSConfigFilePath :: MonadIO m => m FilePath +getAWSConfigFilePath = ( awsConfigFilePath) `liftM` liftIO getHomeDirectory where awsConfigFilePath = ".aws/config" +getAWSCredentialsFilePath:: MonadIO m => m FilePath +getAWSCredentialsFilePath = ( awsCredentialsFilePath) `liftM` liftIO getHomeDirectory + where awsCredentialsFilePath = ".aws/crendetials" + carthageBuildDirectory :: FilePath carthageBuildDirectory = "Carthage" "Build" diff --git a/src/Lib.hs b/src/Lib.hs index 184db4f..a275399 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -31,18 +31,24 @@ import Debug.Trace import qualified Data.ByteString.Char8 as BS (pack) import qualified Data.ByteString.Lazy as LBS import Data.Yaml (encodeFile) +import Data.IORef (newIORef) import Data.Carthage.Cartfile import Data.Carthage.TargetPlatform -import Data.Either.Extra (maybeToEither) +import Data.Either.Extra (maybeToEither, eitherToMaybe) import Data.Maybe (fromMaybe, maybe) import Data.Monoid ((<>)) import Data.Romefile import qualified Data.Map.Strict as M (empty) import qualified Data.Text as T +import qualified Data.Text.Encoding as T (encodeUtf8) import qualified Network.AWS as AWS +import qualified Network.AWS.Auth as AWS (fromEnv) +import qualified Network.AWS.Env as AWS (Env (..), retryConnectionFailure) import qualified Network.AWS.Data as AWS (fromText) +import qualified Network.AWS.Data.Sensitive as AWS (Sensitive (..)) import qualified Network.AWS.S3 as S3 import qualified Network.AWS.Utils as AWS +import qualified Network.HTTP.Conduit as Conduit import Network.URL import System.Directory @@ -66,6 +72,34 @@ s3EndpointOverride (URL (Absolute h) _ _) = S3.s3 s3EndpointOverride _ = S3.s3 +getAWSEnv :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env +getAWSEnv = do + region <- discoverRegion + endpointURL <- runMaybeT . exceptToMaybeT $ discoverEndpoint + (auth, _) <- AWS.catching AWS._MissingEnvError AWS.fromEnv $ \_ -> do + profile <- T.pack . fromMaybe "default" <$> liftIO (lookupEnv (T.unpack "AWS_PROFILE")) + credentilas <- AWS.credentialsFromFile =<< getAWSCredentialsFilePath + let roleARN = eitherToMaybe $ AWS.roleARNOf profile credentilas + case roleARN of + Just role -> do -- There is a role specified so check if there is a source profile + let sourceProfile = eitherToMaybe $ AWS.sourceProfileOf profile credentilas + case sourceProfile of + Just profile -> undefined + Nothing -> undefined + Nothing -> do + let accessKeyId = T.encodeUtf8 <$> AWS.accessKeyIdOf profile credentilas + let secretAccessKey = T.encodeUtf8 <$> AWS.secretAccessKeyOf profile credentilas + let authEnv = AWS.AuthEnv <$> (AWS.AccessKey <$> accessKeyId) + <*> (AWS.Sensitive . AWS.SecretKey <$> secretAccessKey) + <*> Right Nothing + <*> Right Nothing + let auth = (,) <$> (AWS.Auth <$> authEnv) <*> Right (Just region) + ExceptT $ pure auth + manager <- liftIO (Conduit.newManager Conduit.tlsManagerSettings) + ref <- liftIO (newIORef Nothing) + let env = AWS.Env region (\_ _ -> pure ()) (AWS.retryConnectionFailure 3) mempty manager ref auth + return $ AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) env + getAWSRegion :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env getAWSRegion = do region <- discoverRegion @@ -1197,7 +1231,7 @@ discoverRegion = do let eitherEnvRegion = ExceptT . return $ envRegion >>= AWS.fromText . T.pack let eitherFileRegion = - (getS3ConfigFile >>= flip getRegionFromFile (fromMaybe "default" profile)) + (getAWSConfigFilePath >>= flip getRegionFromFile (fromMaybe "default" profile)) `catch` \(e :: IOError) -> ExceptT . return . Left . show $ e eitherEnvRegion <|> eitherFileRegion @@ -1229,7 +1263,7 @@ discoverEndpoint = do profile <- liftIO $ lookupEnv "AWS_PROFILE" let fileEndPointURL = - ( getS3ConfigFile + ( getAWSConfigFilePath >>= flip getEndpointFromFile (fromMaybe "default" profile) ) `catch` \(e :: IOError) -> ExceptT . return . Left . show $ e @@ -1241,10 +1275,31 @@ discoverEndpoint = do -- | Reads an `URL` from a file for a given profile getEndpointFromFile :: MonadIO m - => FilePath -- ^ The path to the file containing the `AWS.Region` - -> String -- ^ The name of the profile to use + => String -- ^ The name of the profile to use + -> FilePath -- ^ The path to the file containing the `AWS.Region` -> ExceptT String m URL -getEndpointFromFile f profile = fromFile f $ \fileContents -> ExceptT . return $ do +getEndpointFromFile profile f = fromFile f $ \fileContents -> ExceptT . return $ do config <- AWS.parseConfigFile fileContents AWS.endPointOf (T.pack profile) config + + +-- -- | Reads `source_profile` from a file for a given profile +-- getSourceProfileFromCredentials +-- :: MonadIO m +-- => String -- ^ The name of the profile to use +-- -> FilePath -- ^ The path to the file containing the `AWS.Region` +-- -> ExceptT String m T.Text +-- getSourceProfileFromCredentials profile f = fromFile f $ \fileContents -> ExceptT . return $ do +-- credentials <- AWS.parseCredentialsFile fileContents +-- AWS.sourceProfileOf (T.pack profile) credentials + +-- -- | Reads `source_profile` from a file for a given profile +-- getRoleARNFromFile +-- :: MonadIO m +-- => String -- ^ The name of the profile to use +-- -> FilePath -- ^ The path to the file containing the `AWS.Region` +-- -> ExceptT String m T.Text +-- getRoleARNFromFile profile f = fromFile f $ \fileContents -> ExceptT . return $ do +-- credentials <- AWS.parseCredentialsFile fileContents +-- AWS.roleARNOf (T.pack profile) credentials \ No newline at end of file diff --git a/src/Network/AWS/Utils.hs b/src/Network/AWS/Utils.hs index 5f8280c..ab73687 100644 --- a/src/Network/AWS/Utils.hs +++ b/src/Network/AWS/Utils.hs @@ -2,12 +2,14 @@ module Network.AWS.Utils ( ConfigFile - , CredentialsFile + , credentialsFromFile , parseConfigFile - , parseCredentialsFile , regionOf , endPointOf , sourceProfileOf + , accessKeyIdOf + , secretAccessKeyOf + , roleARNOf ) where -- For now, only very little information needs to be extracted from the S3 @@ -19,9 +21,12 @@ import Control.Monad ((<=<)) import Data.Either.Utils (maybeToEither) import Data.Ini (Ini, lookupValue, parseIni) import qualified Data.Text as T (Text, null, unpack) +import qualified Data.Text.IO as T (readFile) import qualified Network.AWS as AWS import qualified Network.AWS.Data as AWS import Network.URL +import Control.Monad.IO.Class (MonadIO, liftIO) +import Control.Monad.Except (ExceptT (..), withExceptT) newtype ConfigFile = ConfigFile { _awsConfigIni :: Ini } newtype CredentialsFile = CredentialsFile { _awsCredentialsIni :: Ini } @@ -35,6 +40,17 @@ instance FromIni ConfigFile where instance FromIni CredentialsFile where asIni = _awsCredentialsIni +-- | Reads `CredentialsFile` from a file at a given path +credentialsFromFile + :: MonadIO m + => FilePath -- ^ The path to the file containing the credentials. Usually `~/.aws/credentials` + -> ExceptT String m CredentialsFile +credentialsFromFile filePath = do + file <- liftIO (T.readFile filePath) + withExceptT (("Could not parse " <> filePath <> ": ") <>) (action file) + where + action a = ExceptT . return $ parseCredentialsFile a + regionOf :: T.Text -> ConfigFile -> Either String AWS.Region regionOf profile = parseRegion <=< lookupValue profile "region" . asIni where @@ -54,11 +70,24 @@ endPointOf profile = parseURL <=< lookupValue profile "endpoint" . asIni . T.unpack $ s -sourceProfileOf :: T.Text -> CredentialsFile -> Either String CredentialsFile -sourceProfileOf p credentialsFile = undefined +getPropertyFromCredentials :: T.Text -> T.Text -> CredentialsFile -> Either String T.Text +getPropertyFromCredentials profile property = lookupValue profile property . asIni + +sourceProfileOf :: T.Text -> CredentialsFile -> Either String T.Text +sourceProfileOf profile = getPropertyFromCredentials profile "source_profile" + +roleARNOf :: T.Text -> CredentialsFile -> Either String T.Text +roleARNOf profile = getPropertyFromCredentials profile "role_arn" + +accessKeyIdOf :: T.Text -> CredentialsFile -> Either String T.Text +accessKeyIdOf profile = getPropertyFromCredentials profile "aws_access_key_id" + +secretAccessKeyOf :: T.Text -> CredentialsFile -> Either String T.Text +secretAccessKeyOf profile = getPropertyFromCredentials profile "aws_secret_access_key" parseConfigFile :: T.Text -> Either String ConfigFile parseConfigFile = fmap ConfigFile . parseIni parseCredentialsFile :: T.Text -> Either String CredentialsFile parseCredentialsFile = fmap CredentialsFile . parseIni + From 67faa4c5e47ed9f4750642adf2a62a6634f3e7db Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Thu, 4 Apr 2019 17:54:56 +0200 Subject: [PATCH 03/41] [skip ci] Actually use getAWSEnv --- src/Configuration.hs | 2 +- src/Lib.hs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Configuration.hs b/src/Configuration.hs index d200d07..51f4fda 100644 --- a/src/Configuration.hs +++ b/src/Configuration.hs @@ -39,7 +39,7 @@ getAWSConfigFilePath = ( awsConfigFilePath) `liftM` liftIO getHomeDirectory getAWSCredentialsFilePath:: MonadIO m => m FilePath getAWSCredentialsFilePath = ( awsCredentialsFilePath) `liftM` liftIO getHomeDirectory - where awsCredentialsFilePath = ".aws/crendetials" + where awsCredentialsFilePath = ".aws/credentials" carthageBuildDirectory :: FilePath carthageBuildDirectory = "Carthage" "Build" diff --git a/src/Lib.hs b/src/Lib.hs index a275399..b1098ae 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -434,7 +434,7 @@ getProjectAvailabilityFromCaches [ProjectAvailability] getProjectAvailabilityFromCaches (Just s3BucketName) _ reverseRepositoryMap frameworkVersions platforms = do - env <- lift getAWSRegion + env <- lift getAWSEnv (cachePrefix, _, verbose) <- ask let readerEnv = (env, cachePrefix, verbose) availabilities <- liftIO $ runReaderT @@ -489,7 +489,7 @@ downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersion case (mS3BucketName, mlCacheDir) of (Just s3BucketName, lCacheDir) -> do - env <- lift getAWSRegion + env <- lift getAWSEnv let uploadDownloadEnv = (env, cachePrefix, skipLocalCacheFlag, conconrrentlyFlag, verbose) let action1 = runReaderT @@ -564,7 +564,7 @@ uploadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersions ask case (mS3BucketName, mlCacheDir) of (Just s3BucketName, lCacheDir) -> do - awsEnv <- lift getAWSRegion + awsEnv <- lift getAWSEnv let uploadDownloadEnv = ( awsEnv , cachePrefix From 1d00386a4bc0be907e9b015e000457478a0e068c Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Fri, 5 Apr 2019 11:27:55 +0200 Subject: [PATCH 04/41] [skip ci] remove commented out code --- src/Lib.hs | 28 +++------------------------- 1 file changed, 3 insertions(+), 25 deletions(-) diff --git a/src/Lib.hs b/src/Lib.hs index b1098ae..ecbf8ed 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -91,9 +91,9 @@ getAWSEnv = do let secretAccessKey = T.encodeUtf8 <$> AWS.secretAccessKeyOf profile credentilas let authEnv = AWS.AuthEnv <$> (AWS.AccessKey <$> accessKeyId) <*> (AWS.Sensitive . AWS.SecretKey <$> secretAccessKey) - <*> Right Nothing - <*> Right Nothing - let auth = (,) <$> (AWS.Auth <$> authEnv) <*> Right (Just region) + <*> pure Nothing + <*> pure Nothing + let auth = (,) <$> (AWS.Auth <$> authEnv) <*> pure (pure region) ExceptT $ pure auth manager <- liftIO (Conduit.newManager Conduit.tlsManagerSettings) ref <- liftIO (newIORef Nothing) @@ -1281,25 +1281,3 @@ getEndpointFromFile getEndpointFromFile profile f = fromFile f $ \fileContents -> ExceptT . return $ do config <- AWS.parseConfigFile fileContents AWS.endPointOf (T.pack profile) config - - - --- -- | Reads `source_profile` from a file for a given profile --- getSourceProfileFromCredentials --- :: MonadIO m --- => String -- ^ The name of the profile to use --- -> FilePath -- ^ The path to the file containing the `AWS.Region` --- -> ExceptT String m T.Text --- getSourceProfileFromCredentials profile f = fromFile f $ \fileContents -> ExceptT . return $ do --- credentials <- AWS.parseCredentialsFile fileContents --- AWS.sourceProfileOf (T.pack profile) credentials - --- -- | Reads `source_profile` from a file for a given profile --- getRoleARNFromFile --- :: MonadIO m --- => String -- ^ The name of the profile to use --- -> FilePath -- ^ The path to the file containing the `AWS.Region` --- -> ExceptT String m T.Text --- getRoleARNFromFile profile f = fromFile f $ \fileContents -> ExceptT . return $ do --- credentials <- AWS.parseCredentialsFile fileContents --- AWS.roleARNOf (T.pack profile) credentials \ No newline at end of file From d5e576cb6eea77eb2164b245c4c7babd940591a3 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Fri, 12 Apr 2019 12:05:58 +0200 Subject: [PATCH 05/41] [skip ci] refactor getAWSEnv --- src/Lib.hs | 99 +++++++++++++++++++++++++--------------- src/Network/AWS/Utils.hs | 57 +++++++++++++++++++---- 2 files changed, 109 insertions(+), 47 deletions(-) diff --git a/src/Lib.hs b/src/Lib.hs index ecbf8ed..fb57fb7 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -34,7 +34,8 @@ import Data.Yaml (encodeFile) import Data.IORef (newIORef) import Data.Carthage.Cartfile import Data.Carthage.TargetPlatform -import Data.Either.Extra (maybeToEither, eitherToMaybe) +import Data.Either.Extra (maybeToEither, eitherToMaybe, isRight, mapLeft) +import Data.Either.Utils (fromLeft) import Data.Maybe (fromMaybe, maybe) import Data.Monoid ((<>)) import Data.Romefile @@ -74,31 +75,50 @@ s3EndpointOverride _ = S3.s3 getAWSEnv :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env getAWSEnv = do - region <- discoverRegion - endpointURL <- runMaybeT . exceptToMaybeT $ discoverEndpoint - (auth, _) <- AWS.catching AWS._MissingEnvError AWS.fromEnv $ \_ -> do - profile <- T.pack . fromMaybe "default" <$> liftIO (lookupEnv (T.unpack "AWS_PROFILE")) - credentilas <- AWS.credentialsFromFile =<< getAWSCredentialsFilePath - let roleARN = eitherToMaybe $ AWS.roleARNOf profile credentilas - case roleARN of - Just role -> do -- There is a role specified so check if there is a source profile - let sourceProfile = eitherToMaybe $ AWS.sourceProfileOf profile credentilas - case sourceProfile of - Just profile -> undefined - Nothing -> undefined - Nothing -> do - let accessKeyId = T.encodeUtf8 <$> AWS.accessKeyIdOf profile credentilas - let secretAccessKey = T.encodeUtf8 <$> AWS.secretAccessKeyOf profile credentilas - let authEnv = AWS.AuthEnv <$> (AWS.AccessKey <$> accessKeyId) - <*> (AWS.Sensitive . AWS.SecretKey <$> secretAccessKey) - <*> pure Nothing - <*> pure Nothing - let auth = (,) <$> (AWS.Auth <$> authEnv) <*> pure (pure region) - ExceptT $ pure auth + region <- discoverRegion + endpointURL <- runMaybeT . exceptToMaybeT $ discoverEndpoint + profile <- T.pack . fromMaybe "default" <$> liftIO + (lookupEnv (T.unpack "AWS_PROFILE")) + credentials <- + runExceptT $ AWS.credentialsFromFile =<< getAWSCredentialsFilePath + (auth, _) <- + AWS.catching AWS._MissingEnvError AWS.fromEnv $ \envError -> either + throwError + (\cred -> do + let finalProfile = fromMaybe + profile + (eitherToMaybe $ AWS.sourceProfileOf profile =<< credentials) + let + authAndRegion = + (,) + <$> mapLeft + (\e -> + T.unpack envError + ++ ". " + ++ e + ++ " in file ~/.aws/credentilas" + ) + (AWS.authFromCredentilas finalProfile =<< credentials) + <*> pure (pure region) + liftEither authAndRegion + ) + credentials manager <- liftIO (Conduit.newManager Conduit.tlsManagerSettings) - ref <- liftIO (newIORef Nothing) - let env = AWS.Env region (\_ _ -> pure ()) (AWS.retryConnectionFailure 3) mempty manager ref auth - return $ AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) env + ref <- liftIO (newIORef Nothing) + let roleARN = eitherToMaybe $ AWS.roleARNOf profile =<< credentials + case roleARN of + Just role -> do + undefined -- Make request to STS + Nothing -> + let env = AWS.Env region + (\_ _ -> pure ()) + (AWS.retryConnectionFailure 3) + mempty + manager + ref + auth + in return + $ AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) env getAWSRegion :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env getAWSRegion = do @@ -1231,7 +1251,9 @@ discoverRegion = do let eitherEnvRegion = ExceptT . return $ envRegion >>= AWS.fromText . T.pack let eitherFileRegion = - (getAWSConfigFilePath >>= flip getRegionFromFile (fromMaybe "default" profile)) + ( getAWSConfigFilePath + >>= flip getRegionFromFile (fromMaybe "default" profile) + ) `catch` \(e :: IOError) -> ExceptT . return . Left . show $ e eitherEnvRegion <|> eitherFileRegion @@ -1243,9 +1265,10 @@ getRegionFromFile => FilePath -- ^ The path to the file containing the `AWS.Region` -> String -- ^ The name of the profile to use -> ExceptT String m AWS.Region -getRegionFromFile f profile = fromFile f $ \fileContents -> ExceptT . return $ do - config <- AWS.parseConfigFile fileContents - AWS.regionOf (T.pack profile) config +getRegionFromFile f profile = + fromFile f $ \fileContents -> ExceptT . return $ do + config <- AWS.parseConfigFile fileContents + AWS.regionOf (T.pack profile) config @@ -1261,12 +1284,11 @@ discoverEndpoint = do $ maybeString >>= importURL profile <- liftIO $ lookupEnv "AWS_PROFILE" - let - fileEndPointURL = - ( getAWSConfigFilePath - >>= flip getEndpointFromFile (fromMaybe "default" profile) - ) - `catch` \(e :: IOError) -> ExceptT . return . Left . show $ e + let fileEndPointURL = + ( getAWSConfigFilePath + >>= flip getEndpointFromFile (fromMaybe "default" profile) + ) + `catch` \(e :: IOError) -> ExceptT . return . Left . show $ e (ExceptT . return $ envEndpointURL) <|> fileEndPointURL @@ -1278,6 +1300,7 @@ getEndpointFromFile => String -- ^ The name of the profile to use -> FilePath -- ^ The path to the file containing the `AWS.Region` -> ExceptT String m URL -getEndpointFromFile profile f = fromFile f $ \fileContents -> ExceptT . return $ do - config <- AWS.parseConfigFile fileContents - AWS.endPointOf (T.pack profile) config +getEndpointFromFile profile f = + fromFile f $ \fileContents -> ExceptT . return $ do + config <- AWS.parseConfigFile fileContents + AWS.endPointOf (T.pack profile) config diff --git a/src/Network/AWS/Utils.hs b/src/Network/AWS/Utils.hs index ab73687..65f3499 100644 --- a/src/Network/AWS/Utils.hs +++ b/src/Network/AWS/Utils.hs @@ -3,6 +3,7 @@ module Network.AWS.Utils ( ConfigFile , credentialsFromFile + , authFromCredentilas , parseConfigFile , regionOf , endPointOf @@ -19,11 +20,14 @@ module Network.AWS.Utils import Control.Monad ((<=<)) import Data.Either.Utils (maybeToEither) +import Data.Either.Extra (mapLeft) import Data.Ini (Ini, lookupValue, parseIni) import qualified Data.Text as T (Text, null, unpack) +import qualified Data.Text.Encoding as T (encodeUtf8) import qualified Data.Text.IO as T (readFile) import qualified Network.AWS as AWS import qualified Network.AWS.Data as AWS +import qualified Network.AWS.Data.Sensitive as AWS (Sensitive (..)) import Network.URL import Control.Monad.IO.Class (MonadIO, liftIO) import Control.Monad.Except (ExceptT (..), withExceptT) @@ -41,15 +45,26 @@ instance FromIni CredentialsFile where asIni = _awsCredentialsIni -- | Reads `CredentialsFile` from a file at a given path -credentialsFromFile +credentialsFromFile :: MonadIO m => FilePath -- ^ The path to the file containing the credentials. Usually `~/.aws/credentials` -> ExceptT String m CredentialsFile credentialsFromFile filePath = do file <- liftIO (T.readFile filePath) withExceptT (("Could not parse " <> filePath <> ": ") <>) (action file) - where - action a = ExceptT . return $ parseCredentialsFile a + where action a = ExceptT . return $ parseCredentialsFile a + +authFromCredentilas :: T.Text -> CredentialsFile -> Either String AWS.Auth +authFromCredentilas profile credentials = AWS.Auth <$> authEnv + where + accessKeyId = T.encodeUtf8 <$> accessKeyIdOf profile credentials + secretAccessKey = T.encodeUtf8 <$> secretAccessKeyOf profile credentials + authEnv = + AWS.AuthEnv + <$> (AWS.AccessKey <$> accessKeyId) + <*> (AWS.Sensitive . AWS.SecretKey <$> secretAccessKey) + <*> pure Nothing + <*> pure Nothing regionOf :: T.Text -> ConfigFile -> Either String AWS.Region regionOf profile = parseRegion <=< lookupValue profile "region" . asIni @@ -70,20 +85,44 @@ endPointOf profile = parseURL <=< lookupValue profile "endpoint" . asIni . T.unpack $ s -getPropertyFromCredentials :: T.Text -> T.Text -> CredentialsFile -> Either String T.Text -getPropertyFromCredentials profile property = lookupValue profile property . asIni +getPropertyFromCredentials + :: T.Text -> T.Text -> CredentialsFile -> Either String T.Text +getPropertyFromCredentials profile property = + lookupValue profile property . asIni sourceProfileOf :: T.Text -> CredentialsFile -> Either String T.Text -sourceProfileOf profile = getPropertyFromCredentials profile "source_profile" +sourceProfileOf profile credFile = + getPropertyFromCredentials profile "source_profile" credFile + `withError` const (missingKeyError key profile) + where key = "source_profile" roleARNOf :: T.Text -> CredentialsFile -> Either String T.Text -roleARNOf profile = getPropertyFromCredentials profile "role_arn" +roleARNOf profile credFile = getPropertyFromCredentials profile key credFile + `withError` const (missingKeyError key profile) + where key = "role_arn" accessKeyIdOf :: T.Text -> CredentialsFile -> Either String T.Text -accessKeyIdOf profile = getPropertyFromCredentials profile "aws_access_key_id" +accessKeyIdOf profile credFile = + getPropertyFromCredentials profile key credFile + `withError` const (missingKeyError key profile) + where key = "aws_access_key_id" + +missingKeyError :: T.Text -> T.Text -> String +missingKeyError key profile = + "Could not find key `" + ++ T.unpack key + ++ "` for profile `" + ++ T.unpack profile + ++ "`" + +withError :: Either a b -> (a -> c) -> Either c b +withError = flip mapLeft secretAccessKeyOf :: T.Text -> CredentialsFile -> Either String T.Text -secretAccessKeyOf profile = getPropertyFromCredentials profile "aws_secret_access_key" +secretAccessKeyOf profile credFile = + getPropertyFromCredentials profile key credFile + `withError` const (missingKeyError key profile) + where key = "aws_secret_access_key" parseConfigFile :: T.Text -> Either String ConfigFile parseConfigFile = fmap ConfigFile . parseIni From a0a546c20cf46450b72810d5fa485cdfb76ad7e5 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Sat, 13 Apr 2019 16:30:48 +0200 Subject: [PATCH 06/41] [skip ci] Escalate role_arn with STS --- Rome.cabal | 1 + src/Lib.hs | 49 ++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 39 insertions(+), 11 deletions(-) diff --git a/Rome.cabal b/Rome.cabal index 4e752e0..2cb8dd7 100644 --- a/Rome.cabal +++ b/Rome.cabal @@ -41,6 +41,7 @@ library , amazonka >= 1.6.1 , amazonka-core >= 1.6.1 , amazonka-s3 >= 1.6.1 + , amazonka-sts >= 1.6.1 , exceptions >= 0.8 , lens >= 4.13 , parsec >= 3.1.10 diff --git a/src/Lib.hs b/src/Lib.hs index fb57fb7..ed9db70 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -48,6 +48,7 @@ import qualified Network.AWS.Env as AWS (Env (..), retryConnectionF import qualified Network.AWS.Data as AWS (fromText) import qualified Network.AWS.Data.Sensitive as AWS (Sensitive (..)) import qualified Network.AWS.S3 as S3 +import qualified Network.AWS.STS.AssumeRole as STS (assumeRole, arrsCredentials) import qualified Network.AWS.Utils as AWS import qualified Network.HTTP.Conduit as Conduit @@ -73,6 +74,18 @@ s3EndpointOverride (URL (Absolute h) _ _) = S3.s3 s3EndpointOverride _ = S3.s3 +-- | Tries to get authentication details and region to perform +-- | requests to AWS. +-- | The `AWS_PROFILE` is read from the environment +-- | or falls back to `default`. +-- | The `AWS_REGION` is first read from the environment, if not found +-- | it is read from `~/.aws/config` based on the profile discovered in the previous step. +-- | The `AWS_ACCESS_KEY_ID` & `AWS_SECRET_ACCESS_KEY` are first +-- | read from the environment. If not found, then the `~/.aws/crendetilas` +-- | file is read. If `source_profile` key is present the reading of the +-- | authentication details happens from this profile rather then the `AWS_PROFILE`. +-- | Finally, if `role_arn` is specified, the crendials gathered up to now are used +-- | to obtain new credentials with STS esclated to `role_arn`. getAWSEnv :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env getAWSEnv = do region <- discoverRegion @@ -106,19 +119,33 @@ getAWSEnv = do manager <- liftIO (Conduit.newManager Conduit.tlsManagerSettings) ref <- liftIO (newIORef Nothing) let roleARN = eitherToMaybe $ AWS.roleARNOf profile =<< credentials + let curerntEnv = AWS.Env region + (\_ _ -> pure ()) + (AWS.retryConnectionFailure 3) + mempty + manager + ref + auth case roleARN of - Just role -> do - undefined -- Make request to STS + Just role -> newEnvFromRole role curerntEnv + Nothing -> return + $ AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) curerntEnv + +newEnvFromRole :: MonadIO m => T.Text -> AWS.Env -> ExceptT String m AWS.Env +newEnvFromRole roleARN currentEnv = do + assumeRoleResult <- + liftIO + $ AWS.runResourceT + . AWS.runAWS currentEnv + $ AWS.send + $ STS.assumeRole roleARN "rome-cache-operation" + let maybeAuth = AWS.Auth <$> assumeRoleResult ^. STS.arrsCredentials + case maybeAuth of Nothing -> - let env = AWS.Env region - (\_ _ -> pure ()) - (AWS.retryConnectionFailure 3) - mempty - manager - ref - auth - in return - $ AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) env + throwError + $ "Could not create AWS Auth from STS response: " + ++ show assumeRoleResult + Just newAuth -> return $ currentEnv & AWS.envAuth .~ newAuth getAWSRegion :: (MonadIO m, MonadCatch m) => ExceptT String m AWS.Env getAWSRegion = do From 3492311d4b044f504ce93128ea3e5328dcaa13f0 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Mon, 27 Aug 2018 11:22:27 +0200 Subject: [PATCH 07/41] [skip ci][WIP] Add engine key to RomeCacheInfo -- YAML Only --- src/Data/Romefile.hs | 15 ++++++++++++--- src/Lib.hs | 5 ++++- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/src/Data/Romefile.hs b/src/Data/Romefile.hs index 8739ca8..3812d23 100644 --- a/src/Data/Romefile.hs +++ b/src/Data/Romefile.hs @@ -18,6 +18,7 @@ module Data.Romefile , cacheInfo , bucket , localCacheDir + , enginePath , frameworkName , frameworkType , FrameworkType (..) @@ -174,7 +175,8 @@ cacheInfo :: Lens' Romefile RomeCacheInfo cacheInfo = lens _cacheInfo (\parseResult n -> parseResult { _cacheInfo = n }) data RomeCacheInfo = RomeCacheInfo { _bucket :: Maybe T.Text - , _localCacheDir :: Maybe FilePath -- relative path + , _localCacheDir :: Maybe FilePath -- relative or absolue path + , _enginePath :: Maybe FilePath -- relative or absolue path } deriving (Eq, Show, Generic) @@ -182,11 +184,14 @@ instance FromJSON RomeCacheInfo where parseJSON = withObject "RomeCacheInfo" $ \v -> RomeCacheInfo <$> v .:? "s3Bucket" <*> v .:? "local" + <*> v .:? "engine" instance ToJSON RomeCacheInfo where - toJSON (RomeCacheInfo b l) = object fields + toJSON (RomeCacheInfo b l e) = object fields where - fields = [T.pack "s3Bucket" .= b | isJust b] ++ [T.pack "local" .= l | isJust l] + fields = [T.pack "s3Bucket" .= b | isJust b] + ++ [T.pack "local" .= l | isJust l] + ++ [T.pack "engine" .= e| isJust e] bucket :: Lens' RomeCacheInfo (Maybe T.Text) bucket = lens _bucket (\cInfo n -> cInfo { _bucket = n }) @@ -194,6 +199,9 @@ bucket = lens _bucket (\cInfo n -> cInfo { _bucket = n }) localCacheDir :: Lens' RomeCacheInfo (Maybe FilePath) localCacheDir = lens _localCacheDir (\cInfo n -> cInfo { _localCacheDir = n }) +enginePath :: Lens' RomeCacheInfo (Maybe FilePath) +enginePath = lens _enginePath (\cInfo n -> cInfo { _enginePath = n }) + -- |The canonical name of the Romefile canonicalRomefileName :: String canonicalRomefileName = "Romefile" @@ -226,6 +234,7 @@ toRomefile :: INI.Ini -> Either T.Text Romefile toRomefile ini = do _bucket <- getBucket ini _localCacheDir <- getLocalCacheDir ini + let _engine = Nothing -- Engines are not supported in INI let _repositoryMapEntries = getRepositoryMapEntries ini _ignoreMapEntries = getIgnoreMapEntries ini _cacheInfo = RomeCacheInfo {..} diff --git a/src/Lib.hs b/src/Lib.hs index ed9db70..6b82ece 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -205,6 +205,8 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do let mS3BucketName = S3.BucketName <$> cInfo ^. bucket mlCacheDir <- liftIO $ traverse absolutizePath $ cInfo ^. localCacheDir + mEnginePath <- liftIO $ traverse absolutizePath $ cInfo ^. enginePath + case command of @@ -598,6 +600,7 @@ downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersion uploadArtifacts :: Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> Maybe FilePath -- ^ Just the path to the engine or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. @@ -605,7 +608,7 @@ uploadArtifacts (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) RomeMonad () -uploadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersions platforms +uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap frameworkVersions platforms = do (cachePrefix, skipLocalCacheFlag@(SkipLocalCacheFlag skipLocalCache), concurrentlyFlag@(ConcurrentlyFlag performConcurrently), verbose) <- ask From c924f134ec1784f7fdc9027a5e69923c64536d61 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Mon, 17 Sep 2018 10:43:32 +0200 Subject: [PATCH 08/41] WIP --- src/Lib.hs | 111 +++++++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 91 insertions(+), 20 deletions(-) diff --git a/src/Lib.hs b/src/Lib.hs index 6b82ece..c2f7efa 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -163,11 +163,11 @@ bothCacheKeysMissingMessage conflictingSkipLocalCacheOptionMessage :: String conflictingSkipLocalCacheOptionMessage - = "Error: only \"local\" key is present \ - \in the [Cache] section of your Romefile but you have asked Rome to skip \ + = "Error: only \"local\" defined as cache\ + \in your Romefile, but you have asked Rome to skip \ \this cache." --- | Runs Rome with `RomeOptions` on a given a `AWS.Env`. +-- | Runs Rome with a set of `RomeOptions`. runRomeWithOptions :: RomeOptions -- ^ The `RomeOptions` to run Rome with. -> RomeVersion @@ -204,10 +204,9 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do let cInfo = romeFile ^. cacheInfo let mS3BucketName = S3.BucketName <$> cInfo ^. bucket - mlCacheDir <- liftIO $ traverse absolutizePath $ cInfo ^. localCacheDir + mlCacheDir <- liftIO $ traverse absolutizePath $ cInfo ^. localCacheDir mEnginePath <- liftIO $ traverse absolutizePath $ cInfo ^. enginePath - case command of Upload (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFalg) @@ -284,6 +283,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do (listArtifacts mS3BucketName mlCacheDir + mEnginePath listMode (reverseRepositoryMap <> if _noSkipCurrent noSkipCurrentFlag then currentInvertedMap @@ -318,6 +318,12 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do <> romeVersionToString vers <> noColorControlSequence + -- case (mS3BucketName, mEnginePath) of + -- (Nothing, Nothing) -> undefined -- Proceed donw regular path + -- (Just _, Nothing) -> undefined -- Proceed donw regular path + -- (Just b , Just e) -> throwError $ "Cannot specify both bucket \"" ++ show b ++ "\" and engine at " ++ e ++ " at the same time." + -- (Nothing, Just e) -> undefined -- run command with all info + type FlowFunction = Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. @@ -441,6 +447,7 @@ performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCu listArtifacts :: Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> Maybe FilePath -- ^ Just the path to the engine or Nothing -> ListMode -- ^ A list mode to execute this operation in. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks @@ -450,13 +457,14 @@ listArtifacts (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () -listArtifacts mS3BucketName mlCacheDir listMode reverseRepositoryMap frameworkVersions platforms format +listArtifacts mS3BucketName mlCacheDir mEnginePath listMode reverseRepositoryMap frameworkVersions platforms format = do (_, _, verbose) <- ask let sayFunc = if verbose then sayLnWithTime else sayLn repoAvailabilities <- getProjectAvailabilityFromCaches mS3BucketName mlCacheDir + mEnginePath reverseRepositoryMap frameworkVersions platforms @@ -474,6 +482,7 @@ listArtifacts mS3BucketName mlCacheDir listMode reverseRepositoryMap frameworkVe getProjectAvailabilityFromCaches :: Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> Maybe FilePath -- ^ Just the path to the engine or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform`s to limit the operation to. @@ -481,7 +490,7 @@ getProjectAvailabilityFromCaches (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad [ProjectAvailability] -getProjectAvailabilityFromCaches (Just s3BucketName) _ reverseRepositoryMap frameworkVersions platforms +getProjectAvailabilityFromCaches (Just s3BucketName) _ Nothing reverseRepositoryMap frameworkVersions platforms = do env <- lift getAWSEnv (cachePrefix, _, verbose) <- ask @@ -497,7 +506,7 @@ getProjectAvailabilityFromCaches (Just s3BucketName) _ reverseRepositoryMap fram reverseRepositoryMap availabilities -getProjectAvailabilityFromCaches Nothing (Just lCacheDir) reverseRepositoryMap frameworkVersions platforms +getProjectAvailabilityFromCaches Nothing (Just lCacheDir) Nothing reverseRepositoryMap frameworkVersions platforms = do (cachePrefix, SkipLocalCacheFlag skipLocalCache, _) <- ask when skipLocalCache $ throwError conflictingSkipLocalCacheOptionMessage @@ -511,8 +520,13 @@ getProjectAvailabilityFromCaches Nothing (Just lCacheDir) reverseRepositoryMap f reverseRepositoryMap availabilities -getProjectAvailabilityFromCaches Nothing Nothing _ _ _ = - throwError bothCacheKeysMissingMessage +getProjectAvailabilityFromCaches Nothing lCacheDir (Just ePath) _ _ _ = + undefined-- runEngineList ePath lCacheDir reverseRepositoryMap frameworkVersions platforms +getProjectAvailabilityFromCaches (Just _) _ (Just _) _ _ _ = + throwError conflictingCachesMessage +getProjectAvailabilityFromCaches Nothing Nothing Nothing _ _ _ = + throwError allCacheKeysMissingMessage + @@ -520,6 +534,7 @@ getProjectAvailabilityFromCaches Nothing Nothing _ _ _ = downloadArtifacts :: Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> Maybe FilePath -- ^ Just the path to the engine or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform`s to limit the operation to. @@ -527,7 +542,7 @@ downloadArtifacts (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) RomeMonad () -downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersions platforms +downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap frameworkVersions platforms = do (cachePrefix, skipLocalCacheFlag@(SkipLocalCacheFlag skipLocalCache), conconrrentlyFlag@(ConcurrentlyFlag performConcurrently), verbose) <- ask @@ -535,9 +550,9 @@ downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersion let sayFunc :: MonadIO m => String -> m () sayFunc = if verbose then sayLnWithTime else sayLn - case (mS3BucketName, mlCacheDir) of + case (mS3BucketName, mlCacheDir, mEnginePath) of - (Just s3BucketName, lCacheDir) -> do + (Just s3BucketName, lCacheDir, Nothing) -> do env <- lift getAWSEnv let uploadDownloadEnv = (env, cachePrefix, skipLocalCacheFlag, conconrrentlyFlag, verbose) @@ -559,7 +574,7 @@ downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersion then liftIO $ concurrently_ action1 action2 else liftIO $ action1 >> action2 - (Nothing, Just lCacheDir) -> do + (Nothing, Just lCacheDir, Nothing) -> do let readerEnv = (cachePrefix, verbose) when skipLocalCache $ throwError conflictingSkipLocalCacheOptionMessage @@ -585,8 +600,13 @@ downloadArtifacts mS3BucketName mlCacheDir reverseRepositoryMap frameworkVersion mapM_ (whenLeft sayFunc) errors ) readerEnv - - (Nothing, Nothing) -> throwError bothCacheKeysMissingMessage + -- Use engine + (Nothing, lCacheDir, Just ePath) -> undefined + -- Misconfigured + (Nothing, Nothing , Nothing ) -> throwError allCacheKeysMissingMessage + -- Misconfigured + (Just s3BucketName, Nothing, Just ePath) -> + throwError conflictingCachesMessage where gitRepoNamesAndVersions :: [ProjectNameAndVersion] @@ -612,8 +632,9 @@ uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap framew = do (cachePrefix, skipLocalCacheFlag@(SkipLocalCacheFlag skipLocalCache), concurrentlyFlag@(ConcurrentlyFlag performConcurrently), verbose) <- ask - case (mS3BucketName, mlCacheDir) of - (Just s3BucketName, lCacheDir) -> do + case (mS3BucketName, mlCacheDir, mEnginePath) of + -- S3 Cache, but no engine + (Just s3BucketName, lCacheDir, Nothing) -> do awsEnv <- lift getAWSEnv let uploadDownloadEnv = ( awsEnv @@ -639,8 +660,8 @@ uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap framew if performConcurrently then liftIO $ concurrently_ action1 action2 else liftIO $ action1 >> action2 - - (Nothing, Just lCacheDir) -> do + -- No remotes, just local + (Nothing, Just lCacheDir, Nothing) -> do let readerEnv = (cachePrefix, verbose) when skipLocalCache $ throwError conflictingSkipLocalCacheOptionMessage liftIO @@ -922,6 +943,56 @@ saveFrameworkAndArtifactsToLocalCache lCacheDir reverseRomeMap fVersion@(Framewo + +uploadFrameworksAndArtifactsWithEngine + :: FilePath -- ^ The path to the engine or Nothing + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. + -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .verison files + -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () +uploadFrameworksAndArtifactsWithEngine engine mlCacheDir reverseRomeMap frameworkVersions platforms + = undefined + +uploadFrameworkAndArtifactsWithEngine + :: FilePath -- ^ The path to the engine or Nothing + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. + -> FrameworkVersion -- ^ A`FrameworkVersion` from which to derive Frameworks, dSYMs and .verison files + -> TargetPlatform -- ^ A `TargetPlatform` to restrict this operation to. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () +uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn fwt fwps) _) platform + = do + (cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask + void . runExceptT $ do + frameworkArchive <- createZipArchive frameworkDirectory verbose + unless skipLocalCache + $ maybe (return ()) liftIO + $ runReaderT + <$> ( saveFrameworkToLocalCache + <$> mlCacheDir + <*> Just frameworkArchive + <*> Just reverseRomeMap + <*> Just fVersion + <*> Just platform + ) + <*> Just (cachePrefix, s, verbose) + -- (env, cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask + + -- let uploadDownloadEnv = (env, cachePrefix, verbose) + + -- void . runExceptT $ do + -- frameworkArchive <- createZipArchive frameworkDirectory verbose + where + frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f + platformBuildDirectory = + carthageArtifactsBuildDirectoryForPlatform platform f + frameworkDirectory = + platformBuildDirectory frameworkNameWithFrameworkExtension + dSYMNameWithDSYMExtension = frameworkNameWithFrameworkExtension <> ".dSYM" + dSYMdirectory = platformBuildDirectory dSYMNameWithDSYMExtension + bcSybolMapPath d = platformBuildDirectory bcsymbolmapNameFrom d + -- | Downloads a list of .version files from an S3 Bucket or a local cache. downloadVersionFilesFromCaches :: S3.BucketName -- ^ The cache definition. From 3e6109219d33d872e3fd44a560561d4b72c64c42 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Wed, 24 Oct 2018 16:10:20 +0200 Subject: [PATCH 09/41] Add *.orig to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 93b996f..b3d841a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,7 @@ cabal.sandbox.config *.aux *.hp *.eventlog +*.orig .DS_Store .stack-work/ .vscode/ From 12ad9fbe075ec9ea9dca5ad837902c62f3ac8dd2 Mon Sep 17 00:00:00 2001 From: Tommaso Piazza Date: Sat, 12 Jan 2019 08:43:37 +0100 Subject: [PATCH 10/41] [WIP] Zip intermediates to file --- src/Lib.hs | 81 ++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 58 insertions(+), 23 deletions(-) diff --git a/src/Lib.hs b/src/Lib.hs index c2f7efa..421fd84 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -61,6 +61,12 @@ import Types.Commands as Commands import Utils import Xcode.DWARF +-- # TODO: clean up + +import qualified Codec.Archive.Zip as Zip + +-- # + s3EndpointOverride :: URL -> AWS.Service @@ -156,10 +162,15 @@ getAWSRegion = do <&> AWS.configure (maybe S3.s3 s3EndpointOverride endpointURL) ) -bothCacheKeysMissingMessage :: String -bothCacheKeysMissingMessage - = "Error: expected at least one of \"local\" or \ - \\"S3-Bucket\" key in the [Cache] section of your Romefile." +allCacheKeysMissingMessage :: String +allCacheKeysMissingMessage + = "Error: expected at least one of \"local\", \ + \\"s3Bucket\" or \"engine\" in the cache definition of your Romefile." + +conflictingCachesMessage :: String +conflictingCachesMessage + = "Error: both \"s3Bucket\" and \"engine\" defined. \ + \ Rome cannot use both, choose one." conflictingSkipLocalCacheOptionMessage :: String conflictingSkipLocalCacheOptionMessage @@ -224,6 +235,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do cartfileEntries cachePrefixString mS3BucketName + mEnginePath mlCacheDir platforms @@ -243,6 +255,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do cachePrefixString mS3BucketName mlCacheDir + mEnginePath platforms List (RomeListPayload listMode platforms cachePrefixString printFormat noIgnoreFlag noSkipCurrentFlag) @@ -326,6 +339,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do type FlowFunction = Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing + -> Maybe FilePath -- ^ Just the path to the engine or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. @@ -350,9 +364,10 @@ performWithDefaultFlow -> String -- cachePrefixString -> Maybe S3.BucketName -- mS3BucketName -> Maybe String -- mlCacheDir + -> Maybe String -- mEnginePath -> [TargetPlatform] -- platforms -> RomeMonad () -performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCurrentFlag, concurrentlyFlag) (repositoryMapEntries, ignoreMapEntries, currentMapEntries) gitRepoNames cartfileEntries cachePrefixString mS3BucketName mlCacheDir platforms +performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCurrentFlag, concurrentlyFlag) (repositoryMapEntries, ignoreMapEntries, currentMapEntries) gitRepoNames cartfileEntries cachePrefixString mS3BucketName mlCacheDir mEnginePath platforms = do let ignoreFrameworks = concatMap _frameworks ignoreMapEntries @@ -383,6 +398,7 @@ performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCu (flowFunc mS3BucketName mlCacheDir + mEnginePath reverseRepositoryMap (derivedFrameworkVersions `filterOutFrameworksAndVersionsIfNotIn` finalIgnoreNames @@ -406,6 +422,7 @@ performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCu (flowFunc mS3BucketName mlCacheDir + mEnginePath currentInvertedMap (currentFrameworkVersions `filterOutFrameworksAndVersionsIfNotIn` finalIgnoreNames @@ -437,6 +454,7 @@ performWithDefaultFlow flowFunc (verbose, noIgnoreFlag, skipLocalCache, noSkipCu runReaderT (flowFunc mS3BucketName mlCacheDir + mEnginePath (reverseRepositoryMap <> currentInvertedMap) frameworkVersions platforms @@ -539,7 +557,11 @@ downloadArtifacts -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform`s to limit the operation to. -> ReaderT - (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) + ( CachePrefix + , SkipLocalCacheFlag + , ConcurrentlyFlag + , Bool + ) RomeMonad () downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap frameworkVersions platforms @@ -603,10 +625,9 @@ downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap fram -- Use engine (Nothing, lCacheDir, Just ePath) -> undefined -- Misconfigured - (Nothing, Nothing , Nothing ) -> throwError allCacheKeysMissingMessage + (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage -- Misconfigured - (Just s3BucketName, Nothing, Just ePath) -> - throwError conflictingCachesMessage + (Just s3BucketName, _, Just ePath) -> throwError conflictingCachesMessage where gitRepoNamesAndVersions :: [ProjectNameAndVersion] @@ -625,7 +646,11 @@ uploadArtifacts -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .version files -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. -> ReaderT - (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) + ( CachePrefix + , SkipLocalCacheFlag + , ConcurrentlyFlag + , Bool + ) RomeMonad () uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap frameworkVersions platforms @@ -676,7 +701,7 @@ uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap framew (saveVersionFilesToLocalCache lCacheDir gitRepoNamesAndVersions) readerEnv - (Nothing, Nothing) -> throwError bothCacheKeysMissingMessage + (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage where gitRepoNamesAndVersions :: [ProjectNameAndVersion] gitRepoNamesAndVersions = repoNamesAndVersionForFrameworkVersions @@ -951,8 +976,13 @@ uploadFrameworksAndArtifactsWithEngine -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .verison files -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () -uploadFrameworksAndArtifactsWithEngine engine mlCacheDir reverseRomeMap frameworkVersions platforms - = undefined +uploadFrameworksAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fvs = + mapM_ (sequence . upload) + where + upload = mapM + (uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap) + fvs + uploadFrameworkAndArtifactsWithEngine :: FilePath -- ^ The path to the engine or Nothing @@ -962,7 +992,7 @@ uploadFrameworkAndArtifactsWithEngine -> TargetPlatform -- ^ A `TargetPlatform` to restrict this operation to. -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn fwt fwps) _) platform - = do + = do (cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask void . runExceptT $ do frameworkArchive <- createZipArchive frameworkDirectory verbose @@ -977,21 +1007,26 @@ uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@ <*> Just platform ) <*> Just (cachePrefix, s, verbose) + + liftIO $ saveBinaryToFile + (Zip.fromArchive frameworkArchive) + ("/tmp/" <> frameworkNameWithFrameworkExtension <> ".zip") + -- (env, cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask -- let uploadDownloadEnv = (env, cachePrefix, verbose) -- void . runExceptT $ do -- frameworkArchive <- createZipArchive frameworkDirectory verbose - where - frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f - platformBuildDirectory = - carthageArtifactsBuildDirectoryForPlatform platform f - frameworkDirectory = - platformBuildDirectory frameworkNameWithFrameworkExtension - dSYMNameWithDSYMExtension = frameworkNameWithFrameworkExtension <> ".dSYM" - dSYMdirectory = platformBuildDirectory dSYMNameWithDSYMExtension - bcSybolMapPath d = platformBuildDirectory bcsymbolmapNameFrom d + where + frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f + platformBuildDirectory = + carthageArtifactsBuildDirectoryForPlatform platform f + frameworkDirectory = + platformBuildDirectory frameworkNameWithFrameworkExtension + dSYMNameWithDSYMExtension = frameworkNameWithFrameworkExtension <> ".dSYM" + dSYMdirectory = platformBuildDirectory dSYMNameWithDSYMExtension + bcSybolMapPath d = platformBuildDirectory bcsymbolmapNameFrom d -- | Downloads a list of .version files from an S3 Bucket or a local cache. downloadVersionFilesFromCaches From dd1bae069f4bb0810b22ada43019372ca01159f6 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Fri, 10 May 2019 10:01:25 +0200 Subject: [PATCH 11/41] WIP Probing.hs --- Rome.cabal | 3 ++ src/Engine/Downloading.hs | 87 +++++++++++++++++++++++++++++++++++++++ src/Engine/Probing.hs | 71 ++++++++++++++++++++++++++++++++ src/Engine/Uploading.hs | 1 + src/Lib.hs | 68 ++++++++++++++++++++++++++---- 5 files changed, 223 insertions(+), 7 deletions(-) create mode 100644 src/Engine/Downloading.hs create mode 100644 src/Engine/Probing.hs create mode 100644 src/Engine/Uploading.hs diff --git a/Rome.cabal b/Rome.cabal index 2cb8dd7..c0e952f 100644 --- a/Rome.cabal +++ b/Rome.cabal @@ -36,6 +36,9 @@ library , Caches.Local.Downloading , Caches.Common , Network.AWS.Utils + , Engine.Probing + , Engine.Uploading + , Engine.Downloading build-depends: base >= 4.7 && < 5 , amazonka >= 1.6.1 diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs new file mode 100644 index 0000000..08fb0b9 --- /dev/null +++ b/src/Engine/Downloading.hs @@ -0,0 +1,87 @@ +module Engine.Downloading where + +import qualified Turtle +import Control.Monad.Except +import Caches.Common +import Data.Carthage.TargetPlatform +import System.FilePath +import Utils +import Types hiding ( version ) +import Control.Monad.Reader (ReaderT, ask, runReaderT, + withReaderT) +import qualified Data.Text as T +import qualified Network.AWS as AWS +import qualified Network.AWS.S3 as S3 +import qualified Data.ByteString as BS +import qualified Data.ByteString.Lazy as LBS +import Data.Romefile (Framework (..)) +import Control.Exception (try) +import Control.Lens (view) +import Data.Maybe (fromMaybe) + + +-- | Retrieves a Framework from an S3 Cache and unzip the contents +getFrameworkFromEngine + :: FilePath -- ^ The engine file path + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT + String + (ReaderT (AWS.Env, CachePrefix, Bool) IO) + LBS.ByteString +getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform + = do + (env, CachePrefix prefix, verbose) <- ask + mapExceptT + (withReaderT (const (env, verbose))) + (getArtifactFromEngine enginePath (prefix remoteFrameworkUploadPath) fwn + ) + where + remoteFrameworkUploadPath = + remoteFrameworkPath platform reverseRomeMap f version + + +-- | Retrieves an artifact from an S3 Cache +getArtifactFromEngine + :: FilePath -- ^ The engine file path + -> FilePath -- ^ The path in the cache + -> String -- ^ A colloquial name for the artifact + -> ExceptT String (ReaderT (AWS.Env, Bool) IO) LBS.ByteString +getArtifactFromEngine enginePath remotePath artifactName = undefined --do + -- readerEnv@(_, verbose) <- ask + -- eitherArtifact <- liftIO $ try $ runReaderT + -- (downloadBinaryWithEngine enginePath remotePath artifactName) + -- readerEnv + -- case eitherArtifact of + -- Left e -> + -- throwError + -- $ "Error: could not download " + -- <> artifactName + -- Right artifactBinary -> return artifactBinary + + +-- | Downloads an artificat stored at a given path from an `S3.BucketName`. +downloadBinaryWithEngine + :: FilePath + -> FilePath + -> FilePath + -> ReaderT (AWS.Env, Bool) IO LBS.ByteString +downloadBinaryWithEngine enginePath objectRemotePath objectName = do + (env, verbose) <- ask + do + let sayFunc = if verbose then sayLnWithTime else sayLn + when verbose + $ sayFunc + $ "Started downloading " + <> objectName + <> " from: " + <> objectRemotePath + -- rs <- AWS.send $ S3.getObject enginePath objectKey + -- let contentLength = + -- fromIntegral $ fromMaybe 0 $ view S3.gorsContentLength rs + -- binary <- view S3.gorsBody rs `AWS.sinkBody` sink verbose contentLength + sayFunc $ "Downloaded " <> objectName <> " from: " <> objectRemotePath + return undefined -- return binary + where + objectKey = S3.ObjectKey . T.pack $ objectRemotePath diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs new file mode 100644 index 0000000..e81c4cf --- /dev/null +++ b/src/Engine/Probing.hs @@ -0,0 +1,71 @@ +{-# LANGUAGE OverloadedStrings #-} + +module Engine.Probing where + +import Control.Monad.IO.Class +import Data.Carthage.TargetPlatform +import Data.List (intersect) +import Data.Romefile (_frameworkPlatforms) +import Types hiding (version) +import Utils +import qualified Turtle + + +-- | Probes a `FilePath` to check if each `FrameworkVersion` exists for each `TargetPlatform` +probeEngineForFrameworks + :: MonadIO m + => FilePath -- ^ The engine path. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` to probe for. + -> [TargetPlatform] -- ^ A list target platforms restricting the scope of this action. + -> m [FrameworkAvailability] +probeEngineForFrameworks lCacheDir reverseRomeMap frameworkVersions + = sequence . probeForEachFramework + where + probeForEachFramework = mapM + (probeEngineForFramework lCacheDir reverseRomeMap) + frameworkVersions + + +-- | Probes a `FilePath` to check if a `FrameworkVersion` exists for each `TargetPlatform` +probeEngineForFramework + :: MonadIO m + => FilePath -- ^ The engine path. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` to probe for. + -> [TargetPlatform] -- ^ A list target platforms restricting the scope of this action. + -> m FrameworkAvailability +probeEngineForFramework lCacheDir reverseRomeMap frameworkVersion platforms + = fmap (FrameworkAvailability frameworkVersion) probeForEachPlatform + where + probeForEachPlatform = mapM + (probeEngineForFrameworkOnPlatform lCacheDir + reverseRomeMap + frameworkVersion + ) + (platforms `intersect` (_frameworkPlatforms . _framework $ frameworkVersion)) + + +-- | Probes a `FilePath` to check if a `FrameworkVersion` exists for a given `TargetPlatform` +probeEngineForFrameworkOnPlatform + :: MonadIO m + => FilePath -- ^ The engine path. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` to probe for. + -> TargetPlatform -- ^ A target platforms restricting the scope of this action. + -> m PlatformAvailability +probeEngineForFrameworkOnPlatform enginePath reverseRomeMap (FrameworkVersion fwn version) platform + = do + let cmd = Turtle.fromString $ enginePath + (exitCode) <- Turtle.proc + cmd + ["list", (Turtle.fromString remoteFrameworkUploadPath)] + (return $ Turtle.unsafeTextToLine "") + case exitCode of + Turtle.ExitSuccess -> return () + Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) + return (PlatformAvailability platform True) + where + remoteFrameworkUploadPath = + remoteFrameworkPath platform reverseRomeMap fwn version + diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs new file mode 100644 index 0000000..75e4cd4 --- /dev/null +++ b/src/Engine/Uploading.hs @@ -0,0 +1 @@ +module Engine.Uploading where \ No newline at end of file diff --git a/src/Lib.hs b/src/Lib.hs index b37e8ea..012dbc2 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -18,6 +18,9 @@ import Caches.Local.Uploading import Caches.S3.Downloading import Caches.S3.Probing import Caches.S3.Uploading +import Engine.Downloading +import Engine.Probing +import Engine.Uploading import Configuration import Control.Applicative ((<|>)) import Control.Concurrent.Async.Lifted.Safe (mapConcurrently_, mapConcurrently, concurrently_) @@ -537,8 +540,17 @@ getProjectAvailabilityFromCaches Nothing (Just lCacheDir) Nothing reverseReposit reverseRepositoryMap availabilities -getProjectAvailabilityFromCaches Nothing lCacheDir (Just ePath) _ _ _ = - undefined-- runEngineList ePath lCacheDir reverseRepositoryMap frameworkVersions platforms +getProjectAvailabilityFromCaches Nothing Nothing (Just ePath) reverseRepositoryMap frameworkVersions platforms + = do + let sayFunc = sayLnWithTime + sayFunc $ "Engine path: " <> ePath + availabilities <- probeEngineForFrameworks ePath + reverseRepositoryMap + frameworkVersions + platforms + return $ getMergedGitRepoAvailabilitiesFromFrameworkAvailabilities + reverseRepositoryMap + availabilities getProjectAvailabilityFromCaches (Just _) _ (Just _) _ _ _ = throwError conflictingCachesMessage getProjectAvailabilityFromCaches Nothing Nothing Nothing _ _ _ = @@ -565,7 +577,7 @@ downloadArtifacts () downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap frameworkVersions platforms = do - (cachePrefix, skipLocalCacheFlag@(SkipLocalCacheFlag skipLocalCache), conconrrentlyFlag@(ConcurrentlyFlag performConcurrently), verbose) <- + (cachePrefix, skipLocalCacheFlag@(SkipLocalCacheFlag skipLocalCache), concurrentlyFlag@(ConcurrentlyFlag performConcurrently), verbose) <- ask let sayFunc :: MonadIO m => String -> m () @@ -576,7 +588,7 @@ downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap fram (Just s3BucketName, lCacheDir, Nothing) -> do env <- lift getAWSEnv let uploadDownloadEnv = - (env, cachePrefix, skipLocalCacheFlag, conconrrentlyFlag, verbose) + (env, cachePrefix, skipLocalCacheFlag, concurrentlyFlag, verbose) let action1 = runReaderT (downloadFrameworksAndArtifactsFromCaches s3BucketName lCacheDir @@ -622,7 +634,27 @@ downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap fram ) readerEnv -- Use engine - (Nothing, lCacheDir, Just ePath) -> undefined + (Nothing, lCacheDir, Just ePath) -> do + let readerEnv = (cachePrefix, verbose) + env <- lift getAWSEnv + let uploadDownloadEnv = + (env, cachePrefix, skipLocalCacheFlag, concurrentlyFlag, verbose) + liftIO $ do + runReaderT + (downloadFrameworksAndArtifactsWithEngine ePath + reverseRepositoryMap + frameworkVersions + platforms + ) + uploadDownloadEnv + -- putStrLn (show frameworkVersions) + -- runReaderT + -- (do + -- exitCode <- Turtle.shell "./ciao.sh" Turtle.empty + -- thing <- putStrLn (show exitCode) + -- undefined + -- ) + -- readerEnv -- Misconfigured (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage -- Misconfigured @@ -1290,8 +1322,6 @@ downloadFrameworkAndArtifactsFromCaches s3BucketName (Just lCacheDir) reverseRom dSYMName = fwn <> ".dSYM" frameworkExecutablePath = frameworkBuildBundleForPlatform platform f fwn - - downloadFrameworkAndArtifactsFromCaches s3BucketName Nothing reverseRomeMap fVersion@(FrameworkVersion (Framework fwn _ _) _) platform = do (env, cachePrefix, _, _, verbose) <- ask @@ -1330,6 +1360,30 @@ downloadFrameworkAndArtifactsFromCaches s3BucketName Nothing reverseRomeMap fVer +-- | NEW +downloadFrameworksAndArtifactsWithEngine + :: FilePath -- ^ Just the path to the local cache or Nothing. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. + -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` identifying the Frameworks and dSYMs + -> [TargetPlatform] -- ^ A list of target platforms restricting the scope of this action. + -> ReaderT UploadDownloadCmdEnv IO () +downloadFrameworksAndArtifactsWithEngine mlCacheDir reverseRomeMap fvs platforms + = undefined + + + +-- | NEW +downloadFrameworkAndArtifactsWithEngine + :: FilePath -- ^ Just the path to the local cache or Nothing. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and dSYM + -> TargetPlatform -- ^ A target platforms restricting the scope of this action. + -> ReaderT UploadDownloadCmdEnv IO () +downloadFrameworkAndArtifactsWithEngine lCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform + = undefined + + + -- | Given a `ListMode` and a `ProjectAvailability` produces a `String` -- describing the `ProjectAvailability` for a given `ListMode`. formattedRepoAvailability From 8dc8e00a4020b4b4785ceb1a8e9c9854faf0e893 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 11 May 2019 22:25:25 +0200 Subject: [PATCH 12/41] WIP Engine/Downloading.hs --- src/Caches/S3/Downloading.hs | 2 +- src/Engine/Downloading.hs | 241 +++++++++++++++++++++++++++-------- src/Lib.hs | 99 ++++++++++---- 3 files changed, 263 insertions(+), 79 deletions(-) diff --git a/src/Caches/S3/Downloading.hs b/src/Caches/S3/Downloading.hs index 73ef488..bd4f102 100644 --- a/src/Caches/S3/Downloading.hs +++ b/src/Caches/S3/Downloading.hs @@ -250,7 +250,7 @@ getArtifactFromS3 s3BucketName remotePath artifactName = do --- | Downloads an artificat stored at a given path from an `S3.BucketName`. +-- | Downloads an artifact stored at a given path from an `S3.BucketName`. downloadBinary :: S3.BucketName -> FilePath diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index 08fb0b9..9a44450 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -1,26 +1,33 @@ +{-# LANGUAGE OverloadedStrings #-} + module Engine.Downloading where -import qualified Turtle -import Control.Monad.Except import Caches.Common -import Data.Carthage.TargetPlatform -import System.FilePath -import Utils -import Types hiding ( version ) +import Configuration (carthageArtifactsBuildDirectoryForPlatform) +import Control.Exception (try) +import Control.Lens (view) +import Control.Monad +import Control.Monad.Except import Control.Monad.Reader (ReaderT, ask, runReaderT, withReaderT) -import qualified Data.Text as T -import qualified Network.AWS as AWS -import qualified Network.AWS.S3 as S3 import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as LBS -import Data.Romefile (Framework (..)) -import Control.Exception (try) -import Control.Lens (view) +import Data.Carthage.TargetPlatform +import qualified Data.Conduit as C (ConduitT, await, yield, + (.|)) +import qualified Data.Conduit.Binary as C (sinkLbs) +import Data.Either (lefts) import Data.Maybe (fromMaybe) +import Data.Monoid ((<>)) +import Data.Romefile (Framework (..)) +import qualified Data.Text as T +import System.FilePath (()) +import Types hiding (version) +import Utils +import Xcode.DWARF +import qualified Turtle - --- | Retrieves a Framework from an S3 Cache and unzip the contents +-- | Retrieves a Framework using the engine and unzip the contents getFrameworkFromEngine :: FilePath -- ^ The engine file path -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache @@ -28,60 +35,194 @@ getFrameworkFromEngine -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to -> ExceptT String - (ReaderT (AWS.Env, CachePrefix, Bool) IO) + (ReaderT (CachePrefix, Bool) IO) LBS.ByteString getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform = do - (env, CachePrefix prefix, verbose) <- ask + (CachePrefix prefix, verbose) <- ask mapExceptT - (withReaderT (const (env, verbose))) + (withReaderT (const (verbose))) (getArtifactFromEngine enginePath (prefix remoteFrameworkUploadPath) fwn ) where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap f version +-- | Retrieves a dSYM using the engine +getDSYMFromEngine + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT + String + (ReaderT (CachePrefix, Bool) IO) + LBS.ByteString +getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform + = do + (CachePrefix prefix, verbose) <- ask + let finalRemoteDSYMUploadPath = prefix remoteDSYMUploadPath + mapExceptT (withReaderT (const (verbose))) + $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath dSYMName + where + remoteDSYMUploadPath = remoteDsymPath platform reverseRomeMap f version + dSYMName = fwn <> ".dSYM" + +-- getVersionFileFromEngine +-- | Retrieves a .version file using the engine +getVersionFileFromEngine + :: FilePath + -> ProjectNameAndVersion + -> ExceptT + String + (ReaderT (CachePrefix, Bool) IO) + LBS.ByteString +getVersionFileFromEngine enginePath projectNameAndVersion = do + (CachePrefix prefix, verbose) <- ask + let finalVersionFileRemotePath = prefix versionFileRemotePath + mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine + enginePath + finalVersionFileRemotePath + versionFileName + where + versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion + versionFileRemotePath = remoteVersionFilePath projectNameAndVersion + +-- | Retrieves a bcsymbolmap using the engine +getBcsymbolmapFromEngine + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> DwarfUUID -- ^ The UUID of the bcsymbolmap + -> ExceptT + String + (ReaderT (CachePrefix, Bool) IO) + LBS.ByteString +getBcsymbolmapFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform dwarfUUID + = do + (CachePrefix prefix, verbose) <- ask + let finalRemoteBcsymbolmaploadPath = prefix remoteBcSymbolmapUploadPath + mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine + enginePath + finalRemoteBcsymbolmaploadPath + symbolmapName + where + remoteBcSymbolmapUploadPath = + remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version + symbolmapName = fwn <> "." <> bcsymbolmapNameFrom dwarfUUID + +-- | Retrieves a Framework using the engine and unzip the contents +getAndUnzipFrameworkWithEngine + :: FilePath -- ^ The path to the engine + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT String (ReaderT (CachePrefix, Bool) IO) () +getAndUnzipFrameworkWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) version) platform + = when (platform `elem` fwps) $ do + (_, verbose) <- ask + frameworkBinary <- getFrameworkFromEngine enginePath + reverseRomeMap + fVersion + platform + deleteFrameworkDirectory fVersion platform verbose + unzipBinary frameworkBinary fwn frameworkZipName verbose + <* ifExists + frameworkExecutablePath + (makeExecutable frameworkExecutablePath) + where + frameworkZipName = frameworkArchiveName f version + frameworkExecutablePath = frameworkBuildBundleForPlatform platform f fwn + +-- | Retrieves a dSYM using the engine and unzip the contents +getAndUnzipDSYMWithEngine + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT String (ReaderT (CachePrefix, Bool) IO) () +getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) version) platform + = undefined + -- = when (platform `elem` fwps) $ do + -- (_, _, verbose) <- ask + -- dSYMBinary <- getDSYMFromS3 s3BucketName reverseRomeMap fVersion platform + -- deleteDSYMDirectory fVersion platform verbose + -- unzipBinary dSYMBinary fwn dSYMZipName verbose + -- where dSYMZipName = dSYMArchiveName f version --- | Retrieves an artifact from an S3 Cache +-- | Retrieves all the bcsymbolmap files using the engine and unzip the contents +getAndUnzipBcsymbolmapsWithEngine' + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT + DWARFOperationError + (ReaderT (CachePrefix, Bool) IO) + () +getAndUnzipBcsymbolmapsWithEngine' enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) _) platform + = undefined + -- = when (platform `elem` fwps) $ do + + -- dwarfUUIDs <- withExceptT (const ErrorGettingDwarfUUIDs) + -- $ dwarfUUIDsFrom (frameworkDirectory fwn) + -- eitherDwarfUUIDsOrSucces <- forM + -- dwarfUUIDs + -- (\dwarfUUID -> lift $ runExceptT + -- (withExceptT (\e -> (dwarfUUID, e)) $ getAndUnzipBcsymbolmapFromS3 + -- lCacheDir + -- reverseRomeMap + -- fVersion + -- platform + -- dwarfUUID + -- ) + -- ) + + -- let failedUUIDsAndErrors = lefts eitherDwarfUUIDsOrSucces + -- unless (null failedUUIDsAndErrors) $ throwError $ FailedDwarfUUIDs + -- failedUUIDsAndErrors + -- where + -- frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f + -- platformBuildDirectory = + -- carthageArtifactsBuildDirectoryForPlatform platform f + -- frameworkDirectory = + -- platformBuildDirectory frameworkNameWithFrameworkExtension + +-- | Retrieves an artifact using the engine getArtifactFromEngine :: FilePath -- ^ The engine file path -> FilePath -- ^ The path in the cache -> String -- ^ A colloquial name for the artifact - -> ExceptT String (ReaderT (AWS.Env, Bool) IO) LBS.ByteString -getArtifactFromEngine enginePath remotePath artifactName = undefined --do - -- readerEnv@(_, verbose) <- ask - -- eitherArtifact <- liftIO $ try $ runReaderT - -- (downloadBinaryWithEngine enginePath remotePath artifactName) - -- readerEnv - -- case eitherArtifact of - -- Left e -> - -- throwError - -- $ "Error: could not download " - -- <> artifactName - -- Right artifactBinary -> return artifactBinary - + -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString +getArtifactFromEngine enginePath remotePath artifactName = do + readerEnv@(verbose) <- ask + eitherArtifact <- liftIO $ try $ runReaderT + (downloadBinaryWithEngine enginePath remotePath artifactName) + readerEnv + case eitherArtifact of + Left e -> + throwError + $ "Error: could not download " + <> artifactName + <> " : " + <> awsErrorToString e verbose + Right artifactBinary -> return artifactBinary --- | Downloads an artificat stored at a given path from an `S3.BucketName`. +-- | Downloads an artifact stored at a given path using the engine downloadBinaryWithEngine :: FilePath -> FilePath -> FilePath - -> ReaderT (AWS.Env, Bool) IO LBS.ByteString + -> ReaderT (Bool) IO LBS.ByteString downloadBinaryWithEngine enginePath objectRemotePath objectName = do - (env, verbose) <- ask - do - let sayFunc = if verbose then sayLnWithTime else sayLn - when verbose - $ sayFunc - $ "Started downloading " - <> objectName - <> " from: " - <> objectRemotePath - -- rs <- AWS.send $ S3.getObject enginePath objectKey - -- let contentLength = - -- fromIntegral $ fromMaybe 0 $ view S3.gorsContentLength rs - -- binary <- view S3.gorsBody rs `AWS.sinkBody` sink verbose contentLength - sayFunc $ "Downloaded " <> objectName <> " from: " <> objectRemotePath - return undefined -- return binary - where - objectKey = S3.ObjectKey . T.pack $ objectRemotePath + (verbose) <- ask + let cmd = Turtle.fromString $ enginePath + (exitCode) <- Turtle.proc + cmd + ["download", (Turtle.fromString objectRemotePath), (Turtle.fromString objectName)] + (return $ Turtle.unsafeTextToLine "") + case exitCode of + Turtle.ExitSuccess -> return () + Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) + undefined diff --git a/src/Lib.hs b/src/Lib.hs index 012dbc2..3524212 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -539,11 +539,9 @@ getProjectAvailabilityFromCaches Nothing (Just lCacheDir) Nothing reverseReposit return $ getMergedGitRepoAvailabilitiesFromFrameworkAvailabilities reverseRepositoryMap availabilities - -getProjectAvailabilityFromCaches Nothing Nothing (Just ePath) reverseRepositoryMap frameworkVersions platforms +-- TODO: check how to deal with lCacheDir together with enginePath +getProjectAvailabilityFromCaches Nothing lCacheDir (Just ePath) reverseRepositoryMap frameworkVersions platforms = do - let sayFunc = sayLnWithTime - sayFunc $ "Engine path: " <> ePath availabilities <- probeEngineForFrameworks ePath reverseRepositoryMap frameworkVersions @@ -635,26 +633,16 @@ downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap fram readerEnv -- Use engine (Nothing, lCacheDir, Just ePath) -> do - let readerEnv = (cachePrefix, verbose) - env <- lift getAWSEnv - let uploadDownloadEnv = - (env, cachePrefix, skipLocalCacheFlag, concurrentlyFlag, verbose) + let engineEnv = (cachePrefix, skipLocalCacheFlag, concurrentlyFlag, verbose) liftIO $ do runReaderT (downloadFrameworksAndArtifactsWithEngine ePath + lCacheDir reverseRepositoryMap frameworkVersions platforms ) - uploadDownloadEnv - -- putStrLn (show frameworkVersions) - -- runReaderT - -- (do - -- exitCode <- Turtle.shell "./ciao.sh" Turtle.empty - -- thing <- putStrLn (show exitCode) - -- undefined - -- ) - -- readerEnv + engineEnv -- Misconfigured (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage -- Misconfigured @@ -1177,7 +1165,7 @@ downloadFrameworksAndArtifactsFromCaches s3BucketName mlCacheDir reverseRomeMap -- | Downloads a Framework and it's relative dSYM from and S3 Bucket or a local cache. -- | If the Framework and dSYM are not found in the local cache then they are downloaded from S3. --- | If SkipLocalCache is specified, th local cache is ignored. +-- | If SkipLocalCache is specified, the local cache is ignored. downloadFrameworkAndArtifactsFromCaches :: S3.BucketName -- ^ The cache definition. -> Maybe FilePath -- ^ Just the path to the local cache or Nothing. @@ -1360,29 +1348,84 @@ downloadFrameworkAndArtifactsFromCaches s3BucketName Nothing reverseRomeMap fVer --- | NEW +-- | Downloads a list of Frameworks and relative dSYMs with the engine or a local cache. downloadFrameworksAndArtifactsWithEngine - :: FilePath -- ^ Just the path to the local cache or Nothing. + :: FilePath -- ^ The path to the engine. + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` identifying the Frameworks and dSYMs -> [TargetPlatform] -- ^ A list of target platforms restricting the scope of this action. - -> ReaderT UploadDownloadCmdEnv IO () -downloadFrameworksAndArtifactsWithEngine mlCacheDir reverseRomeMap fvs platforms - = undefined - + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +downloadFrameworksAndArtifactsWithEngine enginePath lCacheDir reverseRomeMap fvs platforms + = do + (_, _, ConcurrentlyFlag performConcurrently, _) <- ask + if performConcurrently + then mapConcurrently_ downloadConcurrently fvs + else mapM_ (sequence . download) platforms + where + downloadConcurrently f = mapConcurrently + (downloadFrameworkAndArtifactsWithEngine enginePath + lCacheDir + reverseRomeMap + f + ) + platforms + download = mapM + (downloadFrameworkAndArtifactsWithEngine enginePath + lCacheDir + reverseRomeMap + ) + fvs --- | NEW +-- | Downloads a Framework and it's relative dSYM with the engine or a local cache. +-- | If the Framework and dSYM are not found in the local cache then they are downloaded using the engine. +-- | If SkipLocalCache is specified, the local cache is ignored. downloadFrameworkAndArtifactsWithEngine - :: FilePath -- ^ Just the path to the local cache or Nothing. + :: FilePath -- ^ The path to the engine. + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and dSYM -> TargetPlatform -- ^ A target platforms restricting the scope of this action. - -> ReaderT UploadDownloadCmdEnv IO () -downloadFrameworkAndArtifactsWithEngine lCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +downloadFrameworkAndArtifactsWithEngine enginePath (Just lCacheDir) reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform = undefined +downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform + = do + (cachePrefix, _, _, verbose) <- ask + + let readerEnv = (cachePrefix, verbose) + + let sayFunc = if verbose then sayLnWithTime else sayLn + eitherError <- liftIO $ runReaderT + (runExceptT $ getAndUnzipFrameworkWithEngine enginePath + reverseRomeMap + fVersion + platform + ) + readerEnv + whenLeft sayFunc eitherError + + eitherDSYMError <- liftIO $ runReaderT + ( runExceptT + $ getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion platform + ) + readerEnv + whenLeft sayFunc eitherDSYMError + eitherSymbolmapsOrErrors <- liftIO $ runReaderT + (runExceptT $ getAndUnzipBcsymbolmapsWithEngine' enginePath + reverseRomeMap + fVersion + platform + ) + readerEnv + flip whenLeft eitherSymbolmapsOrErrors $ \e -> case e of + ErrorGettingDwarfUUIDs -> + sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn + (FailedDwarfUUIDs dwardUUIDsAndErrors) -> + mapM_ (sayFunc . snd) dwardUUIDsAndErrors -- | Given a `ListMode` and a `ProjectAvailability` produces a `String` -- describing the `ProjectAvailability` for a given `ListMode`. From 45a3fda317e3da013a1687a8a6366f88484b726f Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Mon, 13 May 2019 22:14:20 +0200 Subject: [PATCH 13/41] More WIP Engine/Downloading.hs --- src/Engine/Downloading.hs | 196 +++++++++++++++++++++++--------------- 1 file changed, 120 insertions(+), 76 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index 9a44450..da6cbf8 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -39,36 +39,16 @@ getFrameworkFromEngine LBS.ByteString getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform = do - (CachePrefix prefix, verbose) <- ask + (CachePrefix cachePrefix, verbose) <- ask + let frameworkLocalPath = cachePrefix remoteFrameworkUploadPath mapExceptT (withReaderT (const (verbose))) - (getArtifactFromEngine enginePath (prefix remoteFrameworkUploadPath) fwn + (getArtifactFromEngine enginePath remoteFrameworkUploadPath frameworkLocalPath fwn ) where - remoteFrameworkUploadPath = - remoteFrameworkPath platform reverseRomeMap f version + remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap f version --- | Retrieves a dSYM using the engine -getDSYMFromEngine - :: FilePath -- ^ The cache definition - -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache - -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM - -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to - -> ExceptT - String - (ReaderT (CachePrefix, Bool) IO) - LBS.ByteString -getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform - = do - (CachePrefix prefix, verbose) <- ask - let finalRemoteDSYMUploadPath = prefix remoteDSYMUploadPath - mapExceptT (withReaderT (const (verbose))) - $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath dSYMName - where - remoteDSYMUploadPath = remoteDsymPath platform reverseRomeMap f version - dSYMName = fwn <> ".dSYM" --- getVersionFileFromEngine -- | Retrieves a .version file using the engine getVersionFileFromEngine :: FilePath @@ -83,13 +63,15 @@ getVersionFileFromEngine enginePath projectNameAndVersion = do mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalVersionFileRemotePath + finalVersionFileRemotePath versionFileName where versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion versionFileRemotePath = remoteVersionFilePath projectNameAndVersion --- | Retrieves a bcsymbolmap using the engine -getBcsymbolmapFromEngine + +-- | Retrieves a bcsymbolmap with the engine +getBcsymbolmapWithEngine :: FilePath -- ^ The cache definition -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM @@ -99,19 +81,107 @@ getBcsymbolmapFromEngine String (ReaderT (CachePrefix, Bool) IO) LBS.ByteString -getBcsymbolmapFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform dwarfUUID +getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform dwarfUUID = do (CachePrefix prefix, verbose) <- ask let finalRemoteBcsymbolmaploadPath = prefix remoteBcSymbolmapUploadPath mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalRemoteBcsymbolmaploadPath + finalRemoteBcsymbolmaploadPath symbolmapName where remoteBcSymbolmapUploadPath = remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version symbolmapName = fwn <> "." <> bcsymbolmapNameFrom dwarfUUID + +-- | Retrieves a dSYM using the engine +getDSYMFromEngine + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT + String + (ReaderT (CachePrefix, Bool) IO) + LBS.ByteString +getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform + = do + (CachePrefix prefix, verbose) <- ask + let finalRemoteDSYMUploadPath = prefix remoteDSYMUploadPath + mapExceptT (withReaderT (const (verbose))) + $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath finalRemoteDSYMUploadPath dSYMName -- TODO: pass local DSYM FilePath + where + remoteDSYMUploadPath = remoteDsymPath platform reverseRomeMap f version + dSYMName = fwn <> ".dSYM" + + +-- | Retrieves a bcsymbolmap using the engine and unzip the contents +getAndUnzipBcsymbolmapWithEngine + :: FilePath -- ^ The engine path + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> DwarfUUID -- ^ The UUID of the bcsymbolmap + -> ExceptT String (ReaderT (CachePrefix, Bool) IO) () +getAndUnzipBcsymbolmapWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) version) platform dwarfUUID + = when (platform `elem` fwps) $ do + (_, verbose) <- ask + let symbolmapName = fwn <> "." <> bcsymbolmapNameFrom dwarfUUID + binary <- getBcsymbolmapWithEngine enginePath + reverseRomeMap + fVersion + platform + dwarfUUID + deleteFile (bcsymbolmapPath dwarfUUID) verbose + unzipBinary binary symbolmapName (bcsymbolmapZipName dwarfUUID) verbose + where + platformBuildDirectory = + carthageArtifactsBuildDirectoryForPlatform platform f + bcsymbolmapZipName d = bcsymbolmapArchiveName d version + bcsymbolmapPath d = platformBuildDirectory bcsymbolmapNameFrom d + + +-- | Retrieves all the bcsymbolmap files using the engine and unzip the contents +getAndUnzipBcsymbolmapsWithEngine' + :: FilePath -- ^ The cache definition + -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework + -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to + -> ExceptT + DWARFOperationError + (ReaderT (CachePrefix, Bool) IO) + () +getAndUnzipBcsymbolmapsWithEngine' enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) _) platform + -- = undefined + = when (platform `elem` fwps) $ do + + dwarfUUIDs <- withExceptT (const ErrorGettingDwarfUUIDs) + $ dwarfUUIDsFrom (frameworkDirectory fwn) + eitherDwarfUUIDsOrSucces <- forM + dwarfUUIDs + (\dwarfUUID -> lift $ runExceptT + (withExceptT (\e -> (dwarfUUID, e)) $ getAndUnzipBcsymbolmapWithEngine + enginePath + reverseRomeMap + fVersion + platform + dwarfUUID + ) + ) + + let failedUUIDsAndErrors = lefts eitherDwarfUUIDsOrSucces + unless (null failedUUIDsAndErrors) $ throwError $ FailedDwarfUUIDs + failedUUIDsAndErrors + where + frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f + platformBuildDirectory = + carthageArtifactsBuildDirectoryForPlatform platform f + frameworkDirectory = + platformBuildDirectory frameworkNameWithFrameworkExtension + + -- | Retrieves a Framework using the engine and unzip the contents getAndUnzipFrameworkWithEngine :: FilePath -- ^ The path to the engine @@ -135,6 +205,7 @@ getAndUnzipFrameworkWithEngine enginePath reverseRomeMap fVersion@(FrameworkVers frameworkZipName = frameworkArchiveName f version frameworkExecutablePath = frameworkBuildBundleForPlatform platform f fwn + -- | Retrieves a dSYM using the engine and unzip the contents getAndUnzipDSYMWithEngine :: FilePath -- ^ The cache definition @@ -143,59 +214,22 @@ getAndUnzipDSYMWithEngine -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to -> ExceptT String (ReaderT (CachePrefix, Bool) IO) () getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) version) platform - = undefined - -- = when (platform `elem` fwps) $ do - -- (_, _, verbose) <- ask - -- dSYMBinary <- getDSYMFromS3 s3BucketName reverseRomeMap fVersion platform - -- deleteDSYMDirectory fVersion platform verbose - -- unzipBinary dSYMBinary fwn dSYMZipName verbose - -- where dSYMZipName = dSYMArchiveName f version - --- | Retrieves all the bcsymbolmap files using the engine and unzip the contents -getAndUnzipBcsymbolmapsWithEngine' - :: FilePath -- ^ The cache definition - -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache - -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework - -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to - -> ExceptT - DWARFOperationError - (ReaderT (CachePrefix, Bool) IO) - () -getAndUnzipBcsymbolmapsWithEngine' enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) _) platform - = undefined - -- = when (platform `elem` fwps) $ do - - -- dwarfUUIDs <- withExceptT (const ErrorGettingDwarfUUIDs) - -- $ dwarfUUIDsFrom (frameworkDirectory fwn) - -- eitherDwarfUUIDsOrSucces <- forM - -- dwarfUUIDs - -- (\dwarfUUID -> lift $ runExceptT - -- (withExceptT (\e -> (dwarfUUID, e)) $ getAndUnzipBcsymbolmapFromS3 - -- lCacheDir - -- reverseRomeMap - -- fVersion - -- platform - -- dwarfUUID - -- ) - -- ) - - -- let failedUUIDsAndErrors = lefts eitherDwarfUUIDsOrSucces - -- unless (null failedUUIDsAndErrors) $ throwError $ FailedDwarfUUIDs - -- failedUUIDsAndErrors - -- where - -- frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f - -- platformBuildDirectory = - -- carthageArtifactsBuildDirectoryForPlatform platform f - -- frameworkDirectory = - -- platformBuildDirectory frameworkNameWithFrameworkExtension + -- = undefined + = when (platform `elem` fwps) $ do + (_, verbose) <- ask + dSYMBinary <- getDSYMFromEngine enginePath reverseRomeMap fVersion platform + deleteDSYMDirectory fVersion platform verbose + unzipBinary dSYMBinary fwn dSYMZipName verbose + where dSYMZipName = dSYMArchiveName f version -- | Retrieves an artifact using the engine getArtifactFromEngine :: FilePath -- ^ The engine file path - -> FilePath -- ^ The path in the cache + -> FilePath -- ^ The remote path + -> FilePath -- ^ The local path -> String -- ^ A colloquial name for the artifact -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString -getArtifactFromEngine enginePath remotePath artifactName = do +getArtifactFromEngine enginePath remotePath localPath artifactName = do readerEnv@(verbose) <- ask eitherArtifact <- liftIO $ try $ runReaderT (downloadBinaryWithEngine enginePath remotePath artifactName) @@ -218,11 +252,21 @@ downloadBinaryWithEngine downloadBinaryWithEngine enginePath objectRemotePath objectName = do (verbose) <- ask let cmd = Turtle.fromString $ enginePath + let sayFunc = if verbose then sayLnWithTime else sayLn + when verbose + $ sayFunc + $ "Executing script" + <> (show enginePath) + <> " to download " + <> objectName + <> " from: " + <> objectRemotePath (exitCode) <- Turtle.proc cmd - ["download", (Turtle.fromString objectRemotePath), (Turtle.fromString objectName)] + ["download", (Turtle.fromString objectRemotePath), (Turtle.fromString objectRemotePath)] (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) - undefined + -- undefined + liftIO $ LBS.readFile objectRemotePath From a853f168778941e20b7674232b655488d34e1bac Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Mon, 13 May 2019 22:38:23 +0200 Subject: [PATCH 14/41] WIP concurrently Engine/Downloading.hs and Version files --- src/Engine/Downloading.hs | 8 +-- src/Lib.hs | 111 ++++++++++++++++++++++++++++++++++---- 2 files changed, 103 insertions(+), 16 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index da6cbf8..2de972d 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -5,22 +5,15 @@ module Engine.Downloading where import Caches.Common import Configuration (carthageArtifactsBuildDirectoryForPlatform) import Control.Exception (try) -import Control.Lens (view) import Control.Monad import Control.Monad.Except import Control.Monad.Reader (ReaderT, ask, runReaderT, withReaderT) -import qualified Data.ByteString as BS import qualified Data.ByteString.Lazy as LBS import Data.Carthage.TargetPlatform -import qualified Data.Conduit as C (ConduitT, await, yield, - (.|)) -import qualified Data.Conduit.Binary as C (sinkLbs) import Data.Either (lefts) -import Data.Maybe (fromMaybe) import Data.Monoid ((<>)) import Data.Romefile (Framework (..)) -import qualified Data.Text as T import System.FilePath (()) import Types hiding (version) import Utils @@ -270,3 +263,4 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) -- undefined liftIO $ LBS.readFile objectRemotePath + diff --git a/src/Lib.hs b/src/Lib.hs index 3524212..fa7085a 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -634,15 +634,23 @@ downloadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap fram -- Use engine (Nothing, lCacheDir, Just ePath) -> do let engineEnv = (cachePrefix, skipLocalCacheFlag, concurrentlyFlag, verbose) - liftIO $ do - runReaderT - (downloadFrameworksAndArtifactsWithEngine ePath - lCacheDir - reverseRepositoryMap - frameworkVersions - platforms - ) - engineEnv + let action1 = runReaderT + (downloadFrameworksAndArtifactsWithEngine ePath + lCacheDir + reverseRepositoryMap + frameworkVersions + platforms + ) + engineEnv + let action2 = runReaderT + (downloadVersionFilesWithEngine ePath + lCacheDir + gitRepoNamesAndVersions + ) + engineEnv + if performConcurrently + then liftIO $ concurrently_ action1 action2 + else liftIO $ action1 >> action2 -- Misconfigured (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage -- Misconfigured @@ -1427,6 +1435,91 @@ downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersi (FailedDwarfUUIDs dwardUUIDsAndErrors) -> mapM_ (sayFunc . snd) dwardUUIDsAndErrors +-- | Downloads a list of .version files with the engine or a local cache. +downloadVersionFilesWithEngine + :: FilePath -- ^ The engine definition. + -> Maybe FilePath -- ^ Just the local cache path or Nothing + -> [ProjectNameAndVersion] -- ^ A list of `ProjectName`s and `Version`s information. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +downloadVersionFilesWithEngine enginePath lDir = + mapM_ (downloadVersionFileWithEngine enginePath lDir) + + + +-- | Downloads one .version file with the engine or a local cache. +-- | If the .version file is not found in the local cache, it is downloaded with the engine. +-- | If SkipLocalCache is specified, the local cache is ignored. +downloadVersionFileWithEngine + :: FilePath -- ^ The engine definition. + -> Maybe FilePath -- ^ Just the local cache path or Nothing + -> ProjectNameAndVersion -- ^ The `ProjectName` and `Version` information. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +downloadVersionFileWithEngine enginePath (Just lCacheDir) projectNameAndVersion + = do + (cachePrefix@(CachePrefix prefix), SkipLocalCacheFlag skipLocalCache, _, verbose) <- + ask + + when skipLocalCache $ downloadVersionFileWithEngine enginePath + Nothing + projectNameAndVersion + + unless skipLocalCache $ do + eitherSuccess <- runReaderT + (runExceptT $ getAndSaveVersionFileFromLocalCache + lCacheDir + projectNameAndVersion + ) + (cachePrefix, verbose) + case eitherSuccess of + Right _ -> return () + Left e -> liftIO $ do + let sayFunc :: MonadIO m => String -> m () + sayFunc = if verbose then sayLnWithTime else sayLn + sayFunc e + runReaderT + (do + e2 <- runExceptT $ do + versionFileBinary <- getVersionFileFromEngine + enginePath + projectNameAndVersion + saveBinaryToLocalCache lCacheDir + versionFileBinary + (prefix versionFileRemotePath) + versionFileName + verbose + liftIO $ saveBinaryToFile versionFileBinary versionFileLocalPath + sayFunc + $ "Copied " + <> versionFileName + <> " to: " + <> versionFileLocalPath + whenLeft sayFunc e2 + ) + (cachePrefix, verbose) + where + versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion + versionFileLocalPath = carthageBuildDirectory versionFileName + versionFileRemotePath = remoteVersionFilePath projectNameAndVersion + +downloadVersionFileWithEngine enginePath Nothing projectNameAndVersion = do + (cachePrefix, _, _, verbose) <- ask + let sayFunc :: MonadIO m => String -> m () + sayFunc = if verbose then sayLnWithTime else sayLn + eitherError <- liftIO $ runReaderT + (runExceptT $ do + versionFileBinary <- getVersionFileFromEngine enginePath + projectNameAndVersion + liftIO $ saveBinaryToFile versionFileBinary versionFileLocalPath + sayFunc $ "Copied " <> versionFileName <> " to: " <> versionFileLocalPath + ) + (cachePrefix, verbose) + whenLeft sayFunc eitherError + where + versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion + versionFileLocalPath = carthageBuildDirectory versionFileName + + + -- | Given a `ListMode` and a `ProjectAvailability` produces a `String` -- describing the `ProjectAvailability` for a given `ListMode`. formattedRepoAvailability From 0c1f1cd273a5ec4cccff8e76eaf2f5857926215e Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 10:00:13 +0200 Subject: [PATCH 15/41] WIP Engine/Uploading.hs --- src/Engine/Uploading.hs | 82 +++++++++++++++++- src/Lib.hs | 181 +++++++++++++++++++++++++++++++++++----- 2 files changed, 239 insertions(+), 24 deletions(-) diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 75e4cd4..025859f 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -1 +1,81 @@ -module Engine.Uploading where \ No newline at end of file +{-# LANGUAGE OverloadedStrings #-} + +module Engine.Uploading where + +import qualified Codec.Archive.Zip as Zip +import Control.Monad (when) +import Control.Monad.Reader (ReaderT, ask, withReaderT) +import qualified Data.ByteString.Lazy as LBS +import Data.Carthage.TargetPlatform +import Data.Monoid ((<>)) +import Data.Romefile (Framework (..)) +import qualified Data.Text as T +import qualified Network.AWS as AWS +import qualified Network.AWS.S3 as S3 +import System.FilePath (()) +import Types hiding (version) +import Utils +import Xcode.DWARF + + + +-- | Uploads a Framework `Zip.Archive` to an engine. +uploadFrameworkToEngine + :: Zip.Archive -- ^ The `Zip.Archive` of the Framework. + -> FilePath -- ^ The engine definition. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework. + -> TargetPlatform -- ^ A `TargetPlatform`s restricting the scope of this action. + -> ReaderT (CachePrefix, Bool) IO () +uploadFrameworkToEngine frameworkArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = undefined + + + +-- | Uploads a dSYM `Zip.Archive` to an engine. +uploadDsymToEngine + :: Zip.Archive -- ^ The `Zip.Archive` of the dSYM. + -> FilePath -- ^ The engine definition. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. + -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. + -> ReaderT (CachePrefix, Bool) IO () +uploadDsymToEngine dSYMArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = undefined + + + +-- | Uploads a bcsymbolmap `Zip.Archive` to an engine. +uploadBcsymbolmapToEngine + :: DwarfUUID -- ^ The UUID of the bcsymbolmap + -> Zip.Archive -- ^ The `Zip.Archive` of the dSYM. + -> FilePath -- ^ The engine definition. + -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. + -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. + -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. + -> ReaderT (CachePrefix, Bool) IO () +uploadBcsymbolmapToEngine dwarfUUID dwarfArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = undefined + + + +-- | Uploads a .version file to an engine +uploadVersionFileToEngine' + :: FilePath -- ^ The engine definition. + -> LBS.ByteString -- ^ The contents of the .version file. + -> ProjectNameAndVersion -- ^ The information used to derive the name and path for the .version file. + -> ReaderT (CachePrefix, Bool) IO () +uploadVersionFileToEngine' s3BucketName versionFileContent projectNameAndVersion = undefined + + + +-- | Uploads an artifact to an engine +uploadBinary + :: AWS.ToBody a + => S3.BucketName + -> a + -> FilePath + -> FilePath + -> ReaderT (Bool) IO () +uploadBinary s3BucketName binaryZip destinationPath objectName = undefined + diff --git a/src/Lib.hs b/src/Lib.hs index fa7085a..1cd175a 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -222,26 +222,27 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do case command of - Upload (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFalg) - -> sayVersionWarning romeVersion verbose + Upload (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFlag) + -> do + sayVersionWarning romeVersion verbose *> performWithDefaultFlow uploadArtifacts ( verbose , noIgnoreFlag , skipLocalCache , noSkipCurrentFlag - , concurrentlyFalg + , concurrentlyFlag ) (repositoryMapEntries, ignoreMapEntries, currentMapEntries) gitRepoNames cartfileEntries cachePrefixString mS3BucketName - mEnginePath mlCacheDir + mEnginePath platforms - Download (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFalg) + Download (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFlag) -> sayVersionWarning romeVersion verbose *> performWithDefaultFlow downloadArtifacts @@ -249,7 +250,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do , noIgnoreFlag , skipLocalCache , noSkipCurrentFlag - , concurrentlyFalg + , concurrentlyFlag ) (repositoryMapEntries, ignoreMapEntries, currentMapEntries) gitRepoNames @@ -727,7 +728,31 @@ uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap framew >> runReaderT (saveVersionFilesToLocalCache lCacheDir gitRepoNamesAndVersions) readerEnv - + -- Engine, maybe Cache + (Nothing, lCacheDir, Just enginePath) -> do + let engineEnv = + ( cachePrefix + , skipLocalCacheFlag + , concurrentlyFlag + , verbose + ) + let action1 = runReaderT + (uploadFrameworksAndArtifactsToEngine enginePath + lCacheDir + reverseRepositoryMap + frameworkVersions + platforms + ) + engineEnv + let action2 = runReaderT + (uploadVersionFilesToEngine enginePath + lCacheDir + gitRepoNamesAndVersions + ) + engineEnv + if performConcurrently + then liftIO $ concurrently_ action1 action2 + else liftIO $ action1 >> action2 (Nothing, Nothing, Nothing) -> throwError allCacheKeysMissingMessage where gitRepoNamesAndVersions :: [ProjectNameAndVersion] @@ -736,6 +761,47 @@ uploadArtifacts mS3BucketName mlCacheDir mEnginePath reverseRepositoryMap framew frameworkVersions +-- | Uploads a lest of .version files to the engine. +uploadVersionFilesToEngine + :: FilePath -- ^ The engine definition. + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing. + -> [ProjectNameAndVersion] -- ^ A list of `ProjectName` and `Version` information. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +uploadVersionFilesToEngine enginePath mlCacheDir = + mapM_ (uploadVersionFileToEngine enginePath mlCacheDir) + + +-- | Uploads a .version file to the engine. +uploadVersionFileToEngine + :: FilePath -- ^ The engine definition. + -> Maybe FilePath -- ^ Just the path to the local cache or Nothing. + -> ProjectNameAndVersion -- ^ The information used to derive the name and path for the .version file. + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +uploadVersionFileToEngine enginePath mlCacheDir projectNameAndVersion = do + (cachePrefix, SkipLocalCacheFlag skipLocalCache, _, verbose) <- ask + + versionFileExists <- liftIO $ doesFileExist versionFileLocalPath + + when versionFileExists $ do + versionFileContent <- liftIO $ LBS.readFile versionFileLocalPath + unless skipLocalCache + $ maybe (return ()) liftIO + $ saveVersionFileBinaryToLocalCache + <$> mlCacheDir + <*> Just cachePrefix + <*> Just versionFileContent + <*> Just projectNameAndVersion + <*> Just verbose + liftIO $ runReaderT + (uploadVersionFileToEngine' enginePath + versionFileContent + projectNameAndVersion + ) + (cachePrefix, verbose) + where + + versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion + versionFileLocalPath = carthageBuildDirectory versionFileName -- | Uploads a lest of .version files to the caches. uploadVersionFilesToCaches @@ -996,18 +1062,28 @@ saveFrameworkAndArtifactsToLocalCache lCacheDir reverseRomeMap fVersion@(Framewo -uploadFrameworksAndArtifactsWithEngine +uploadFrameworksAndArtifactsToEngine :: FilePath -- ^ The path to the engine or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` from which to derive Frameworks, dSYMs and .verison files -> [TargetPlatform] -- ^ A list of `TargetPlatform` to restrict this operation to. - -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () -uploadFrameworksAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fvs = - mapM_ (sequence . upload) + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +uploadFrameworksAndArtifactsToEngine enginePath mlCacheDir reverseRomeMap fvs platforms = do + (_, _, ConcurrentlyFlag performConcurrently, _) <- ask + if performConcurrently + then mapConcurrently_ uploadConcurrently fvs + else mapM_ (sequence . upload) platforms where + uploadConcurrently f = mapConcurrently + (uploadFrameworkAndArtifactsWithEngine enginePath + mlCacheDir + reverseRomeMap + f + ) + platforms upload = mapM - (uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap) + (uploadFrameworkAndArtifactsWithEngine enginePath mlCacheDir reverseRomeMap) fvs @@ -1017,10 +1093,11 @@ uploadFrameworkAndArtifactsWithEngine -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `ProjectName`s. -> FrameworkVersion -- ^ A`FrameworkVersion` from which to derive Frameworks, dSYMs and .verison files -> TargetPlatform -- ^ A `TargetPlatform` to restrict this operation to. - -> ReaderT (CachePrefix, SkipLocalCacheFlag, Bool) RomeMonad () -uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn fwt fwps) _) platform + -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () +uploadFrameworkAndArtifactsWithEngine enginePath mlCacheDir reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn fwt fwps) _) platform = do - (cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask + (cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), ConcurrentlyFlag performConcurrently, verbose) <- ask + let readerEnv = (cachePrefix, verbose) void . runExceptT $ do frameworkArchive <- createZipArchive frameworkDirectory verbose unless skipLocalCache @@ -1035,17 +1112,73 @@ uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@ ) <*> Just (cachePrefix, s, verbose) - liftIO $ saveBinaryToFile - (Zip.fromArchive frameworkArchive) - ("/tmp/" <> frameworkNameWithFrameworkExtension <> ".zip") + liftIO $ runReaderT + (uploadFrameworkToEngine frameworkArchive + enginePath + reverseRomeMap + fVersion + platform + ) + readerEnv + + void . runExceptT $ do + dSYMArchive <- createZipArchive dSYMdirectory verbose + unless skipLocalCache + $ maybe (return ()) liftIO + $ runReaderT + <$> ( saveDsymToLocalCache + <$> mlCacheDir + <*> Just dSYMArchive + <*> Just reverseRomeMap + <*> Just fVersion + <*> Just platform + ) + <*> Just (cachePrefix, s, verbose) + liftIO $ runReaderT + (uploadDsymToEngine dSYMArchive + enginePath + reverseRomeMap + fVersion + platform + ) + readerEnv - -- (env, cachePrefix, s@(SkipLocalCacheFlag skipLocalCache), verbose) <- ask + void . runExceptT $ do + dwarfUUIDs <- dwarfUUIDsFrom (frameworkDirectory fwn) + maybeUUIDsArchives <- liftIO $ forM dwarfUUIDs $ \dwarfUUID -> + runMaybeT $ do + dwarfArchive <- exceptToMaybeT + $ createZipArchive (bcSymbolMapPath dwarfUUID) verbose + return (dwarfUUID, dwarfArchive) - -- let uploadDownloadEnv = (env, cachePrefix, verbose) + unless skipLocalCache + $ forM_ maybeUUIDsArchives + $ mapM + $ \(dwarfUUID, dwarfArchive) -> + maybe (return ()) liftIO + $ runReaderT + <$> ( saveBcsymbolmapToLocalCache + <$> mlCacheDir + <*> Just dwarfUUID + <*> Just dwarfArchive + <*> Just reverseRomeMap + <*> Just fVersion + <*> Just platform + ) + <*> Just (cachePrefix, s, verbose) - -- void . runExceptT $ do - -- frameworkArchive <- createZipArchive frameworkDirectory verbose + forM_ maybeUUIDsArchives $ mapM $ \(dwarfUUID, dwarfArchive) -> + liftIO $ runReaderT + (uploadBcsymbolmapToEngine dwarfUUID + dwarfArchive + enginePath + reverseRomeMap + fVersion + platform + ) + readerEnv where + frameworkNameWithFrameworkExtension = appendFrameworkExtensionTo f platformBuildDirectory = carthageArtifactsBuildDirectoryForPlatform platform f @@ -1053,7 +1186,9 @@ uploadFrameworkAndArtifactsWithEngine engine mlCacheDir reverseRomeMap fVersion@ platformBuildDirectory frameworkNameWithFrameworkExtension dSYMNameWithDSYMExtension = frameworkNameWithFrameworkExtension <> ".dSYM" dSYMdirectory = platformBuildDirectory dSYMNameWithDSYMExtension - bcSybolMapPath d = platformBuildDirectory bcsymbolmapNameFrom d + bcSymbolMapPath d = platformBuildDirectory bcsymbolmapNameFrom d + + -- | Downloads a list of .version files from an S3 Bucket or a local cache. downloadVersionFilesFromCaches From f5f32edc3c99ba33783730fd46c4d398d6a23d3c Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 11:13:08 +0200 Subject: [PATCH 16/41] Fix typos --- src/Caches/Local/Uploading.hs | 2 +- src/Caches/S3/Uploading.hs | 2 +- src/Utils.hs | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Caches/Local/Uploading.hs b/src/Caches/Local/Uploading.hs index f50e69f..9cb55fa 100644 --- a/src/Caches/Local/Uploading.hs +++ b/src/Caches/Local/Uploading.hs @@ -96,7 +96,7 @@ saveBinaryToLocalCache -> LBS.ByteString -- ^ The `ByteString` to save. -> FilePath -- ^ The destination path inside the base directory. -> String -- ^ A colloquial name for the artifact printed when verbose is `True`. - -> Bool -- ^ A verbostiry flag. + -> Bool -- ^ A verbosity flag. -> m () saveBinaryToLocalCache cachePath binaryZip destinationPath objectName verbose = do diff --git a/src/Caches/S3/Uploading.hs b/src/Caches/S3/Uploading.hs index d0e1469..d310c01 100644 --- a/src/Caches/S3/Uploading.hs +++ b/src/Caches/S3/Uploading.hs @@ -103,7 +103,7 @@ uploadVersionFileToS3 s3BucketName versionFileContent projectNameAndVersion = --- | Uploads an artificat to an `S3.BucketName` at a given path in the bucket. +-- | Uploads an artifact to an `S3.BucketName` at a given path in the bucket. uploadBinary :: AWS.ToBody a => S3.BucketName diff --git a/src/Utils.hs b/src/Utils.hs index 602149f..1247793 100644 --- a/src/Utils.hs +++ b/src/Utils.hs @@ -741,7 +741,7 @@ whenLeft _ (Right _) = return () --- | Read a file as `Text` and pefrom an action +-- | Read a file as `Text` and perform an action fromFile :: MonadIO m => FilePath -- ^ The `FilePath` to the file to read From ea400fdbd6605c956cf8e423cd3e989b2402ed82 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 11:13:25 +0200 Subject: [PATCH 17/41] WIP Engine/Uploading.hs --- src/Engine/Downloading.hs | 1 - src/Engine/Uploading.hs | 82 ++++++++++++++++++++++++++++++++------- src/Lib.hs | 1 - 3 files changed, 67 insertions(+), 17 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index 2de972d..a4adfff 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -261,6 +261,5 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do case exitCode of Turtle.ExitSuccess -> return () Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) - -- undefined liftIO $ LBS.readFile objectRemotePath diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 025859f..f401a6e 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -5,6 +5,7 @@ module Engine.Uploading where import qualified Codec.Archive.Zip as Zip import Control.Monad (when) import Control.Monad.Reader (ReaderT, ask, withReaderT) +import Control.Monad.IO.Class import qualified Data.ByteString.Lazy as LBS import Data.Carthage.TargetPlatform import Data.Monoid ((<>)) @@ -16,7 +17,7 @@ import System.FilePath (()) import Types hiding (version) import Utils import Xcode.DWARF - +import qualified Turtle -- | Uploads a Framework `Zip.Archive` to an engine. @@ -27,8 +28,17 @@ uploadFrameworkToEngine -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework. -> TargetPlatform -- ^ A `TargetPlatform`s restricting the scope of this action. -> ReaderT (CachePrefix, Bool) IO () -uploadFrameworkToEngine frameworkArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform - = undefined +uploadFrameworkToEngine frameworkArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = when (platform `elem` fwps) $ do + (CachePrefix prefix, verbose) <- ask + withReaderT (const (verbose)) $ uploadBinary + enginePath + (Zip.fromArchive frameworkArchive) + (prefix remoteFrameworkUploadPath) + fwn + where + remoteFrameworkUploadPath = + remoteFrameworkPath platform reverseRomeMap f version @@ -40,8 +50,15 @@ uploadDsymToEngine -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. -> ReaderT (CachePrefix, Bool) IO () -uploadDsymToEngine dSYMArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform - = undefined +uploadDsymToEngine dSYMArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = when (platform `elem` fwps) $ do + (CachePrefix prefix, verbose) <- ask + withReaderT (const (verbose)) $ uploadBinary + enginePath + (Zip.fromArchive dSYMArchive) + (prefix remoteDsymUploadPath) + (fwn <> ".dSYM") + where remoteDsymUploadPath = remoteDsymPath platform reverseRomeMap f version @@ -54,9 +71,17 @@ uploadBcsymbolmapToEngine -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. -> ReaderT (CachePrefix, Bool) IO () -uploadBcsymbolmapToEngine dwarfUUID dwarfArchive s3BucketName reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform - = undefined - +uploadBcsymbolmapToEngine dwarfUUID dwarfArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform + = when (platform `elem` fwps) $ do + (CachePrefix prefix, verbose) <- ask + withReaderT (const (verbose)) $ uploadBinary + enginePath + (Zip.fromArchive dwarfArchive) + (prefix remoteBcsymbolmapUploadPath) + (fwn <> "." <> bcsymbolmapNameFrom dwarfUUID) + where + remoteBcsymbolmapUploadPath = + remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version -- | Uploads a .version file to an engine @@ -65,17 +90,44 @@ uploadVersionFileToEngine' -> LBS.ByteString -- ^ The contents of the .version file. -> ProjectNameAndVersion -- ^ The information used to derive the name and path for the .version file. -> ReaderT (CachePrefix, Bool) IO () -uploadVersionFileToEngine' s3BucketName versionFileContent projectNameAndVersion = undefined +uploadVersionFileToEngine' enginePath versionFileContent projectNameAndVersion = + do + (CachePrefix prefix, verbose) <- ask + withReaderT (const (verbose)) $ uploadBinary + enginePath + versionFileContent + (prefix versionFileRemotePath) + versionFileName + where + versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion + versionFileRemotePath = remoteVersionFilePath projectNameAndVersion -- | Uploads an artifact to an engine uploadBinary - :: AWS.ToBody a - => S3.BucketName - -> a + :: MonadIO a + => FilePath + -> LBS.ByteString -> FilePath -> FilePath - -> ReaderT (Bool) IO () -uploadBinary s3BucketName binaryZip destinationPath objectName = undefined - + -> ReaderT (Bool) a () +uploadBinary enginePath binaryZip destinationPath objectName = + do + (verbose) <- ask + let cmd = Turtle.fromString $ enginePath + sayLn + $ "Executing script " + <> (show enginePath) + <> " to upload " + <> objectName + <> " from: " + <> destinationPath + (exitCode) <- Turtle.proc + cmd + ["upload", (Turtle.fromString destinationPath), (Turtle.fromString destinationPath)] + (return $ Turtle.unsafeTextToLine "") + case exitCode of + Turtle.ExitSuccess -> return () + Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) + liftIO $ saveBinaryToFile binaryZip destinationPath diff --git a/src/Lib.hs b/src/Lib.hs index 1cd175a..a9e8c3a 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -1111,7 +1111,6 @@ uploadFrameworkAndArtifactsWithEngine enginePath mlCacheDir reverseRomeMap fVers <*> Just platform ) <*> Just (cachePrefix, s, verbose) - liftIO $ runReaderT (uploadFrameworkToEngine frameworkArchive enginePath From f32774f102dca7e3fddbe78da5102cbd002ed4ae Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 17:16:20 +0200 Subject: [PATCH 18/41] Correctly parse Probe command exit code to assume existence of framework --- src/Engine/Probing.hs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs index e81c4cf..311c3ef 100644 --- a/src/Engine/Probing.hs +++ b/src/Engine/Probing.hs @@ -62,9 +62,9 @@ probeEngineForFrameworkOnPlatform enginePath reverseRomeMap (FrameworkVersion fw ["list", (Turtle.fromString remoteFrameworkUploadPath)] (return $ Turtle.unsafeTextToLine "") case exitCode of - Turtle.ExitSuccess -> return () - Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) - return (PlatformAvailability platform True) + -- If script exists with success, we assume the framework exists. + Turtle.ExitSuccess -> return (PlatformAvailability platform True) + Turtle.ExitFailure n -> return (PlatformAvailability platform False) --Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap fwn version From 19cf5b1628aadd6f5fc1ea785ce0ce453b4a04e2 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 17:37:22 +0200 Subject: [PATCH 19/41] Fix writing file before invoking the engine --- src/Engine/Uploading.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index f401a6e..724e864 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -116,6 +116,7 @@ uploadBinary enginePath binaryZip destinationPath objectName = do (verbose) <- ask let cmd = Turtle.fromString $ enginePath + liftIO $ saveBinaryToFile binaryZip destinationPath sayLn $ "Executing script " <> (show enginePath) @@ -130,4 +131,3 @@ uploadBinary enginePath binaryZip destinationPath objectName = case exitCode of Turtle.ExitSuccess -> return () Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) - liftIO $ saveBinaryToFile binaryZip destinationPath From e4a1045621354088a0873890477c65cdc27993f9 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 21:46:49 +0200 Subject: [PATCH 20/41] WIP in not crashing when script fails to download --- src/Engine/Downloading.hs | 42 +++++++++++++++++++++++++++++++-------- src/Engine/Uploading.hs | 2 +- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index a4adfff..339ebfe 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -14,6 +14,7 @@ import Data.Carthage.TargetPlatform import Data.Either (lefts) import Data.Monoid ((<>)) import Data.Romefile (Framework (..)) +import System.Directory import System.FilePath (()) import Types hiding (version) import Utils @@ -26,13 +27,12 @@ getFrameworkFromEngine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to - -> ExceptT - String - (ReaderT (CachePrefix, Bool) IO) - LBS.ByteString + -> ExceptT String (ReaderT (CachePrefix, Bool) IO) LBS.ByteString getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform = do (CachePrefix cachePrefix, verbose) <- ask + -- frameworkExistsInLocalCache <- + -- liftIO . doesFileExist $ frameworkLocalCachePath prefix let frameworkLocalPath = cachePrefix remoteFrameworkUploadPath mapExceptT (withReaderT (const (verbose))) @@ -41,6 +41,27 @@ getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap f version + -- frameworkExistsInLocalCache <- + -- liftIO . doesFileExist $ frameworkLocalCachePath prefix + -- if frameworkExistsInLocalCache + -- then + -- liftIO + -- . runResourceT + -- . C.runConduit + -- $ C.sourceFile (frameworkLocalCachePath prefix) + -- C..| C.sinkLbs + -- else + -- throwError + -- $ "Error: could not find " + -- <> fwn + -- <> " in local cache at : " + -- <> frameworkLocalCachePath prefix + -- where + -- frameworkLocalCachePath cPrefix = + -- lCacheDir cPrefix remoteFrameworkUploadPath + -- remoteFrameworkUploadPath = + -- remoteFrameworkPath platform reverseRomeMap f version + -- | Retrieves a .version file using the engine getVersionFileFromEngine @@ -78,6 +99,7 @@ getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framewor = do (CachePrefix prefix, verbose) <- ask let finalRemoteBcsymbolmaploadPath = prefix remoteBcSymbolmapUploadPath + sayLnWithTime (show dwarfUUID) mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalRemoteBcsymbolmaploadPath @@ -241,7 +263,7 @@ downloadBinaryWithEngine :: FilePath -> FilePath -> FilePath - -> ReaderT (Bool) IO LBS.ByteString + -> (ReaderT (Bool) IO) LBS.ByteString downloadBinaryWithEngine enginePath objectRemotePath objectName = do (verbose) <- ask let cmd = Turtle.fromString $ enginePath @@ -260,6 +282,10 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () - Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) - liftIO $ LBS.readFile objectRemotePath - + Turtle.ExitFailure n -> sayFunc + $ "Error: could not download " + <> objectRemotePath + binaryExists <- liftIO . doesFileExist $ objectRemotePath + if binaryExists + then liftIO $ LBS.readFile objectRemotePath + else liftIO $ LBS.readFile objectRemotePath -- TODO: throwError in some way diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 724e864..09e04f5 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -130,4 +130,4 @@ uploadBinary enginePath binaryZip destinationPath objectName = (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () - Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) + Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) -- TODO: print out error and don't crash From 8f33e61cea434b8afa8350cd8ac09c385e31f47f Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 22:31:39 +0200 Subject: [PATCH 21/41] Throw custom error when file is not there --- src/Engine/Downloading.hs | 4 ++-- src/Engine/Uploading.hs | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index 339ebfe..fe3a742 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -4,7 +4,7 @@ module Engine.Downloading where import Caches.Common import Configuration (carthageArtifactsBuildDirectoryForPlatform) -import Control.Exception (try) +import Control.Exception (try, throwIO) import Control.Monad import Control.Monad.Except import Control.Monad.Reader (ReaderT, ask, runReaderT, @@ -288,4 +288,4 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do binaryExists <- liftIO . doesFileExist $ objectRemotePath if binaryExists then liftIO $ LBS.readFile objectRemotePath - else liftIO $ LBS.readFile objectRemotePath -- TODO: throwError in some way + else liftIO . throwIO $ userError "ooops" diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 09e04f5..bdfebb0 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -117,7 +117,7 @@ uploadBinary enginePath binaryZip destinationPath objectName = (verbose) <- ask let cmd = Turtle.fromString $ enginePath liftIO $ saveBinaryToFile binaryZip destinationPath - sayLn + sayLn -- TODO: probably remove this debugging log if not verbose, or maybe always? $ "Executing script " <> (show enginePath) <> " to upload " @@ -130,4 +130,8 @@ uploadBinary enginePath binaryZip destinationPath objectName = (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () - Turtle.ExitFailure n -> Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) -- TODO: print out error and don't crash + Turtle.ExitFailure n -> do + sayLn + $ "Error: could not upload " + <> destinationPath + From e7acde4a91653497cdaac420ab2e4a6024a65ef7 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Tue, 14 May 2019 22:49:01 +0200 Subject: [PATCH 22/41] WIP --- src/Lib.hs | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Lib.hs b/src/Lib.hs index a9e8c3a..00898b3 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -1538,16 +1538,21 @@ downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersi (cachePrefix, _, _, verbose) <- ask let readerEnv = (cachePrefix, verbose) - let sayFunc = if verbose then sayLnWithTime else sayLn - eitherError <- liftIO $ runReaderT - (runExceptT $ getAndUnzipFrameworkWithEngine enginePath - reverseRomeMap - fVersion - platform - ) - readerEnv - whenLeft sayFunc eitherError + + liftIO $ do + runReaderT + (do + errors <- + mapM runExceptT + $ getAndUnzipFrameworkWithEngine + enginePath + reverseRomeMap + fVersion + platform + mapM_ (whenLeft sayFunc) errors + ) + readerEnv eitherDSYMError <- liftIO $ runReaderT ( runExceptT From d532c720cb0f3a32be2b2806f7b844c980413516 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Wed, 15 May 2019 00:30:34 +0200 Subject: [PATCH 23/41] =?UTF-8?q?Don=E2=80=99t=20fail=20when=20file=20is?= =?UTF-8?q?=20not=20there?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Engine/Downloading.hs | 28 ++--------------- src/Lib.hs | 65 +++++++++++++++++++-------------------- 2 files changed, 33 insertions(+), 60 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index fe3a742..a92d2a0 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -31,8 +31,6 @@ getFrameworkFromEngine getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ _) version) platform = do (CachePrefix cachePrefix, verbose) <- ask - -- frameworkExistsInLocalCache <- - -- liftIO . doesFileExist $ frameworkLocalCachePath prefix let frameworkLocalPath = cachePrefix remoteFrameworkUploadPath mapExceptT (withReaderT (const (verbose))) @@ -41,27 +39,6 @@ getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap f version - -- frameworkExistsInLocalCache <- - -- liftIO . doesFileExist $ frameworkLocalCachePath prefix - -- if frameworkExistsInLocalCache - -- then - -- liftIO - -- . runResourceT - -- . C.runConduit - -- $ C.sourceFile (frameworkLocalCachePath prefix) - -- C..| C.sinkLbs - -- else - -- throwError - -- $ "Error: could not find " - -- <> fwn - -- <> " in local cache at : " - -- <> frameworkLocalCachePath prefix - -- where - -- frameworkLocalCachePath cPrefix = - -- lCacheDir cPrefix remoteFrameworkUploadPath - -- remoteFrameworkUploadPath = - -- remoteFrameworkPath platform reverseRomeMap f version - -- | Retrieves a .version file using the engine getVersionFileFromEngine @@ -229,7 +206,6 @@ getAndUnzipDSYMWithEngine -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to -> ExceptT String (ReaderT (CachePrefix, Bool) IO) () getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) version) platform - -- = undefined = when (platform `elem` fwps) $ do (_, verbose) <- ask dSYMBinary <- getDSYMFromEngine enginePath reverseRomeMap fVersion platform @@ -255,7 +231,7 @@ getArtifactFromEngine enginePath remotePath localPath artifactName = do $ "Error: could not download " <> artifactName <> " : " - <> awsErrorToString e verbose + <> awsErrorToString e verbose -- TODO: change it here Right artifactBinary -> return artifactBinary -- | Downloads an artifact stored at a given path using the engine @@ -288,4 +264,4 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do binaryExists <- liftIO . doesFileExist $ objectRemotePath if binaryExists then liftIO $ LBS.readFile objectRemotePath - else liftIO . throwIO $ userError "ooops" + else fail "ooooooooops" diff --git a/src/Lib.hs b/src/Lib.hs index 00898b3..8e6ec76 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -1531,7 +1531,7 @@ downloadFrameworkAndArtifactsWithEngine -> TargetPlatform -- ^ A target platforms restricting the scope of this action. -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () downloadFrameworkAndArtifactsWithEngine enginePath (Just lCacheDir) reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform - = undefined + = undefined -- TODO: implement cache retrieval downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform = do @@ -1540,39 +1540,36 @@ downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersi let readerEnv = (cachePrefix, verbose) let sayFunc = if verbose then sayLnWithTime else sayLn - liftIO $ do - runReaderT - (do - errors <- - mapM runExceptT - $ getAndUnzipFrameworkWithEngine - enginePath - reverseRomeMap - fVersion - platform - mapM_ (whenLeft sayFunc) errors - ) - readerEnv - - eitherDSYMError <- liftIO $ runReaderT - ( runExceptT - $ getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion platform - ) - readerEnv - whenLeft sayFunc eitherDSYMError - - eitherSymbolmapsOrErrors <- liftIO $ runReaderT - (runExceptT $ getAndUnzipBcsymbolmapsWithEngine' enginePath - reverseRomeMap - fVersion - platform - ) - readerEnv - flip whenLeft eitherSymbolmapsOrErrors $ \e -> case e of - ErrorGettingDwarfUUIDs -> - sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn - (FailedDwarfUUIDs dwardUUIDsAndErrors) -> - mapM_ (sayFunc . snd) dwardUUIDsAndErrors + let a = runReaderT + (do + error <- + runExceptT + $ getAndUnzipFrameworkWithEngine + enginePath + reverseRomeMap + fVersion + platform + whenLeft sayFunc error + eitherDSYMError <- + runExceptT + $ getAndUnzipDSYMWithEngine + enginePath + reverseRomeMap + fVersion + platform + whenLeft sayFunc eitherDSYMError + eitherSymbolmapsOrErrors <- runExceptT $ getAndUnzipBcsymbolmapsWithEngine' enginePath + reverseRomeMap + fVersion + platform + flip whenLeft eitherSymbolmapsOrErrors $ \e -> case e of + ErrorGettingDwarfUUIDs -> + sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn + (FailedDwarfUUIDs dwardUUIDsAndErrors) -> + mapM_ (sayFunc . snd) dwardUUIDsAndErrors + ) + readerEnv + liftIO $ a `catch` \e -> print ("Caught " ++ show (e :: IOError)) -- | Downloads a list of .version files with the engine or a local cache. downloadVersionFilesWithEngine From 43a1ad026db8f25e2bfc566e233848289f5bafd0 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Wed, 15 May 2019 09:20:18 +0200 Subject: [PATCH 24/41] Remove catch and specify signature of eitherArtifact that we need --- src/Engine/Downloading.hs | 7 +++-- src/Lib.hs | 60 +++++++++++++++++++-------------------- 2 files changed, 34 insertions(+), 33 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index a92d2a0..f8285ad 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -1,4 +1,5 @@ {-# LANGUAGE OverloadedStrings #-} +{-# LANGUAGE ScopedTypeVariables #-} module Engine.Downloading where @@ -222,7 +223,7 @@ getArtifactFromEngine -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString getArtifactFromEngine enginePath remotePath localPath artifactName = do readerEnv@(verbose) <- ask - eitherArtifact <- liftIO $ try $ runReaderT + eitherArtifact :: Either IOError LBS.ByteString <- liftIO $ try $ runReaderT (downloadBinaryWithEngine enginePath remotePath artifactName) readerEnv case eitherArtifact of @@ -231,7 +232,7 @@ getArtifactFromEngine enginePath remotePath localPath artifactName = do $ "Error: could not download " <> artifactName <> " : " - <> awsErrorToString e verbose -- TODO: change it here + <> show e Right artifactBinary -> return artifactBinary -- | Downloads an artifact stored at a given path using the engine @@ -264,4 +265,4 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do binaryExists <- liftIO . doesFileExist $ objectRemotePath if binaryExists then liftIO $ LBS.readFile objectRemotePath - else fail "ooooooooops" + else fail "Binary was not downloaded by engine" diff --git a/src/Lib.hs b/src/Lib.hs index 8e6ec76..50008f5 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -1540,36 +1540,36 @@ downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersi let readerEnv = (cachePrefix, verbose) let sayFunc = if verbose then sayLnWithTime else sayLn - let a = runReaderT - (do - error <- - runExceptT - $ getAndUnzipFrameworkWithEngine - enginePath - reverseRomeMap - fVersion - platform - whenLeft sayFunc error - eitherDSYMError <- - runExceptT - $ getAndUnzipDSYMWithEngine - enginePath - reverseRomeMap - fVersion - platform - whenLeft sayFunc eitherDSYMError - eitherSymbolmapsOrErrors <- runExceptT $ getAndUnzipBcsymbolmapsWithEngine' enginePath - reverseRomeMap - fVersion - platform - flip whenLeft eitherSymbolmapsOrErrors $ \e -> case e of - ErrorGettingDwarfUUIDs -> - sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn - (FailedDwarfUUIDs dwardUUIDsAndErrors) -> - mapM_ (sayFunc . snd) dwardUUIDsAndErrors - ) - readerEnv - liftIO $ a `catch` \e -> print ("Caught " ++ show (e :: IOError)) + liftIO $ + runReaderT + (do + error <- + runExceptT + $ getAndUnzipFrameworkWithEngine + enginePath + reverseRomeMap + fVersion + platform + whenLeft sayFunc error + eitherDSYMError <- + runExceptT + $ getAndUnzipDSYMWithEngine + enginePath + reverseRomeMap + fVersion + platform + whenLeft sayFunc eitherDSYMError + eitherSymbolmapsOrErrors <- runExceptT $ getAndUnzipBcsymbolmapsWithEngine' enginePath + reverseRomeMap + fVersion + platform + flip whenLeft eitherSymbolmapsOrErrors $ \e -> case e of + ErrorGettingDwarfUUIDs -> + sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn + (FailedDwarfUUIDs dwardUUIDsAndErrors) -> + mapM_ (sayFunc . snd) dwardUUIDsAndErrors + ) + readerEnv -- | Downloads a list of .version files with the engine or a local cache. downloadVersionFilesWithEngine From 4a3e2224cb9054f90e772481b550d9b79475a99d Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Wed, 15 May 2019 15:06:53 +0200 Subject: [PATCH 25/41] Implement local cache together with engine --- src/Engine/Downloading.hs | 1 - src/Lib.hs | 134 +++++++++++++++++++++++++++++++++++++- 2 files changed, 133 insertions(+), 2 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index f8285ad..c8f959c 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -77,7 +77,6 @@ getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framewor = do (CachePrefix prefix, verbose) <- ask let finalRemoteBcsymbolmaploadPath = prefix remoteBcSymbolmapUploadPath - sayLnWithTime (show dwarfUUID) mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalRemoteBcsymbolmaploadPath diff --git a/src/Lib.hs b/src/Lib.hs index 50008f5..18187f5 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -1531,7 +1531,138 @@ downloadFrameworkAndArtifactsWithEngine -> TargetPlatform -- ^ A target platforms restricting the scope of this action. -> ReaderT (CachePrefix, SkipLocalCacheFlag, ConcurrentlyFlag, Bool) IO () downloadFrameworkAndArtifactsWithEngine enginePath (Just lCacheDir) reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform - = undefined -- TODO: implement cache retrieval + = do + (cachePrefix@(CachePrefix prefix), SkipLocalCacheFlag skipLocalCache, _, verbose) <- ask + + let readerEnv = (cachePrefix, verbose) + + when skipLocalCache $ downloadFrameworkAndArtifactsWithEngine + enginePath + Nothing + reverseRomeMap + fVersion + platform + + unless skipLocalCache $ do + eitherFrameworkSuccess <- runReaderT + (runExceptT $ getAndUnzipFrameworkFromLocalCache lCacheDir + reverseRomeMap + fVersion + platform + ) + readerEnv + let sayFunc :: MonadIO m => String -> m () + sayFunc = if verbose then sayLnWithTime else sayLn + + case eitherFrameworkSuccess of + Right _ -> return () + Left e -> liftIO $ do + sayFunc e + runReaderT + (do + e2 <- runExceptT $ do + frameworkBinary <- getFrameworkFromEngine enginePath + reverseRomeMap + fVersion + platform + saveBinaryToLocalCache lCacheDir + frameworkBinary + (prefix remoteFrameworkUploadPath) + fwn + verbose + deleteFrameworkDirectory fVersion platform verbose + unzipBinary frameworkBinary fwn frameworkZipName verbose + <* ifExists + frameworkExecutablePath + (makeExecutable frameworkExecutablePath) + whenLeft sayFunc e2 + ) + readerEnv + + eitherBcsymbolmapsOrErrors <- runReaderT + (runExceptT $ getAndUnzipBcsymbolmapsFromLocalCache' lCacheDir + reverseRomeMap + fVersion + platform + ) + readerEnv + case eitherBcsymbolmapsOrErrors of + Right _ -> return () + Left ErrorGettingDwarfUUIDs -> + sayFunc $ "Error: Cannot retrieve symbolmaps ids for " <> fwn + Left (FailedDwarfUUIDs dwardUUIDsAndErrors) -> do + mapM_ (sayFunc . snd) dwardUUIDsAndErrors + forM_ (map fst dwardUUIDsAndErrors) + $ \dwarfUUID -> liftIO $ runReaderT + (do + e <- runExceptT $ do + let symbolmapLoggingName = + fwn <> "." <> bcsymbolmapNameFrom dwarfUUID + let bcsymbolmapZipName d = bcsymbolmapArchiveName d version + let localBcsymbolmapPathFrom d = + platformBuildDirectory bcsymbolmapNameFrom d + symbolmapBinary <- getBcsymbolmapWithEngine enginePath + reverseRomeMap + fVersion + platform + dwarfUUID + saveBinaryToLocalCache + lCacheDir + symbolmapBinary + (prefix remoteBcSymbolmapUploadPathFromDwarf dwarfUUID + ) + fwn + verbose + deleteFile (localBcsymbolmapPathFrom dwarfUUID) verbose + unzipBinary symbolmapBinary + symbolmapLoggingName + (bcsymbolmapZipName dwarfUUID) + verbose + whenLeft sayFunc e + ) + readerEnv + + eitherDSYMSuccess <- runReaderT + (runExceptT $ getAndUnzipDSYMFromLocalCache lCacheDir + reverseRomeMap + fVersion + platform + ) + readerEnv + case eitherDSYMSuccess of + Right _ -> return () + Left e -> liftIO $ do + sayFunc e + runReaderT + (do + e2 <- runExceptT $ do + dSYMBinary <- getDSYMFromEngine enginePath + reverseRomeMap + fVersion + platform + saveBinaryToLocalCache lCacheDir + dSYMBinary + (prefix remotedSYMUploadPath) + dSYMName + verbose + deleteDSYMDirectory fVersion platform verbose + unzipBinary dSYMBinary dSYMName dSYMZipName verbose + whenLeft sayFunc e2 + ) + readerEnv + where + frameworkZipName = frameworkArchiveName f version + remoteFrameworkUploadPath = + remoteFrameworkPath platform reverseRomeMap f version + remoteBcSymbolmapUploadPathFromDwarf dwarfUUID = + remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version + dSYMZipName = dSYMArchiveName f version + remotedSYMUploadPath = remoteDsymPath platform reverseRomeMap f version + platformBuildDirectory = + carthageArtifactsBuildDirectoryForPlatform platform f + dSYMName = fwn <> ".dSYM" + frameworkExecutablePath = frameworkBuildBundleForPlatform platform f fwn + downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ _) version) platform = do @@ -1571,6 +1702,7 @@ downloadFrameworkAndArtifactsWithEngine enginePath Nothing reverseRomeMap fVersi ) readerEnv + -- | Downloads a list of .version files with the engine or a local cache. downloadVersionFilesWithEngine :: FilePath -- ^ The engine definition. From 28577f456d9b35aae3d910ec3b6a8cad97d3370c Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Wed, 15 May 2019 17:19:46 +0200 Subject: [PATCH 26/41] Fix warnings and remove unused local path --- src/Engine/Downloading.hs | 11 ++++------- src/Engine/Probing.hs | 4 ++-- src/Lib.hs | 18 ++++-------------- 3 files changed, 10 insertions(+), 23 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index c8f959c..61ee3d9 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -5,7 +5,7 @@ module Engine.Downloading where import Caches.Common import Configuration (carthageArtifactsBuildDirectoryForPlatform) -import Control.Exception (try, throwIO) +import Control.Exception (try) import Control.Monad import Control.Monad.Except import Control.Monad.Reader (ReaderT, ask, runReaderT, @@ -35,7 +35,7 @@ getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework let frameworkLocalPath = cachePrefix remoteFrameworkUploadPath mapExceptT (withReaderT (const (verbose))) - (getArtifactFromEngine enginePath remoteFrameworkUploadPath frameworkLocalPath fwn + (getArtifactFromEngine enginePath frameworkLocalPath fwn ) where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap f version @@ -55,7 +55,6 @@ getVersionFileFromEngine enginePath projectNameAndVersion = do mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalVersionFileRemotePath - finalVersionFileRemotePath versionFileName where versionFileName = versionFileNameForProjectName $ fst projectNameAndVersion @@ -80,7 +79,6 @@ getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framewor mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine enginePath finalRemoteBcsymbolmaploadPath - finalRemoteBcsymbolmaploadPath symbolmapName where remoteBcSymbolmapUploadPath = @@ -103,7 +101,7 @@ getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ (CachePrefix prefix, verbose) <- ask let finalRemoteDSYMUploadPath = prefix remoteDSYMUploadPath mapExceptT (withReaderT (const (verbose))) - $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath finalRemoteDSYMUploadPath dSYMName -- TODO: pass local DSYM FilePath + $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath dSYMName where remoteDSYMUploadPath = remoteDsymPath platform reverseRomeMap f version dSYMName = fwn <> ".dSYM" @@ -217,10 +215,9 @@ getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f getArtifactFromEngine :: FilePath -- ^ The engine file path -> FilePath -- ^ The remote path - -> FilePath -- ^ The local path -> String -- ^ A colloquial name for the artifact -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString -getArtifactFromEngine enginePath remotePath localPath artifactName = do +getArtifactFromEngine enginePath remotePath artifactName = do readerEnv@(verbose) <- ask eitherArtifact :: Either IOError LBS.ByteString <- liftIO $ try $ runReaderT (downloadBinaryWithEngine enginePath remotePath artifactName) diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs index 311c3ef..be3ec92 100644 --- a/src/Engine/Probing.hs +++ b/src/Engine/Probing.hs @@ -62,9 +62,9 @@ probeEngineForFrameworkOnPlatform enginePath reverseRomeMap (FrameworkVersion fw ["list", (Turtle.fromString remoteFrameworkUploadPath)] (return $ Turtle.unsafeTextToLine "") case exitCode of - -- If script exists with success, we assume the framework exists. + -- If script exits with success, we assume the framework exists. Turtle.ExitSuccess -> return (PlatformAvailability platform True) - Turtle.ExitFailure n -> return (PlatformAvailability platform False) --Turtle.die (cmd <> " failed with exit code: " <> Turtle.repr n) + Turtle.ExitFailure n -> return (PlatformAvailability platform False) where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap fwn version diff --git a/src/Lib.hs b/src/Lib.hs index 18187f5..7114939 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -36,19 +36,16 @@ import Data.Yaml (encodeFile) import Data.IORef (newIORef) import Data.Carthage.Cartfile import Data.Carthage.TargetPlatform -import Data.Either.Extra (maybeToEither, eitherToMaybe, isRight, mapLeft) -import Data.Either.Utils (fromLeft) +import Data.Either.Extra (maybeToEither, eitherToMaybe, mapLeft) import Data.Maybe (fromMaybe, maybe) import Data.Monoid ((<>)) import Data.Romefile import qualified Data.Map.Strict as M (empty) import qualified Data.Text as T -import qualified Data.Text.Encoding as T (encodeUtf8) import qualified Network.AWS as AWS import qualified Network.AWS.Auth as AWS (fromEnv) import qualified Network.AWS.Env as AWS (Env (..), retryConnectionFailure) import qualified Network.AWS.Data as AWS (fromText) -import qualified Network.AWS.Data.Sensitive as AWS (Sensitive (..)) import qualified Network.AWS.S3 as S3 import qualified Network.AWS.STS.AssumeRole as STS (assumeRole, arrsCredentials) import qualified Network.AWS.Utils as AWS @@ -63,13 +60,6 @@ import Types.Commands as Commands import Utils import Xcode.DWARF --- # TODO: clean up - -import qualified Codec.Archive.Zip as Zip - --- # - - s3EndpointOverride :: URL -> AWS.Service s3EndpointOverride (URL (Absolute h) _ _) = @@ -540,8 +530,8 @@ getProjectAvailabilityFromCaches Nothing (Just lCacheDir) Nothing reverseReposit return $ getMergedGitRepoAvailabilitiesFromFrameworkAvailabilities reverseRepositoryMap availabilities --- TODO: check how to deal with lCacheDir together with enginePath -getProjectAvailabilityFromCaches Nothing lCacheDir (Just ePath) reverseRepositoryMap frameworkVersions platforms + +getProjectAvailabilityFromCaches Nothing _ (Just ePath) reverseRepositoryMap frameworkVersions platforms = do availabilities <- probeEngineForFrameworks ePath reverseRepositoryMap @@ -1621,7 +1611,7 @@ downloadFrameworkAndArtifactsWithEngine enginePath (Just lCacheDir) reverseRomeM whenLeft sayFunc e ) readerEnv - + eitherDSYMSuccess <- runReaderT (runExceptT $ getAndUnzipDSYMFromLocalCache lCacheDir reverseRomeMap From c4da191c5544575aecca4b929d520ed01a676f6e Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Wed, 15 May 2019 23:07:14 +0200 Subject: [PATCH 27/41] Add first integration test --- .../dynamic-frameworks-engine-yml.bats | 93 +++++++++++++++++++ integration-tests/engine.sh | 34 +++++++ 2 files changed, 127 insertions(+) create mode 100644 integration-tests/dynamic-frameworks-engine-yml.bats create mode 100755 integration-tests/engine.sh diff --git a/integration-tests/dynamic-frameworks-engine-yml.bats b/integration-tests/dynamic-frameworks-engine-yml.bats new file mode 100644 index 0000000..4e5721c --- /dev/null +++ b/integration-tests/dynamic-frameworks-engine-yml.bats @@ -0,0 +1,93 @@ +#!/usr/bin/env bats + +setup() { + + export ALAMOFIRE_VERSION="4.7.3" + export RESULT_VERSION="4.0.0" + + cd $BATS_TMPDIR + + rm -rf Rome-Tests + + mkdir Rome-Tests && cd Rome-Tests + + if [ "$BATS_TEST_NUMBER" -eq 1 ]; then + printf "github \"Alamofire/Alamofire\" == ${ALAMOFIRE_VERSION}\n" > Cartfile + printf "github \"antitypical/Result\" == ${RESULT_VERSION}\n" >> Cartfile + + # carthage bootstrap --cache-builds --no-use-binaries + + rm -rf ../_Carthage_build_bkp + cp -R /Users/balestrapatrick/Desktop/integration-test/ . + cp -R Carthage/Build/ ../_Carthage_build_bkp + + rm -f ../_Cartfile_bkp + rm -f ../_Cartfile.resolved_bkp + cp Cartfile ../_Cartfile_bkp + cp Cartfile.resolved ../_Cartfile.resolved_bkp + + else + mkdir -p Carthage/Build + cp -R ../_Carthage_build_bkp/ Carthage/Build + cp ../_Cartfile_bkp Cartfile + cp ../_Cartfile.resolved_bkp Cartfile.resolved + fi + + cat >> Romefile << EOF +cache: + local: rome-local-cache + engine: engine.sh +ignoreMap: + - Alamofire: + - name: Alamofire + platforms: [Mac] +EOF + + IOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/iOS/Alamofire.framework/Alamofire)) + TVOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/tvOS/Alamofire.framework/Alamofire)) + WATCHOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/watchOS/Alamofire.framework/Alamofire)) + + export ALAMOFIRE_IOS_ARMV7_DWARF_UUID=${IOS_DWARFDUMP_OUT[9]} + export ALAMOFIRE_IOS_ARM64_DWARF_UUID=${IOS_DWARFDUMP_OUT[13]} + export ALAMOFIRE_TVOS_ARM64_DWARF_UUID=${TVOS_DWARFDUMP_OUT[5]} + export ALAMOFIRE_WATCHOS_ARMV7K_DWARF_UUID=${WATCHOS_DWARFDUMP_OUT[5]} + + IOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/iOS/Result.framework/Result)) + TVOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/tvOS/Result.framework/Result)) + WATCHOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/watchOS/Result.framework/Result)) + + export RESULT_IOS_ARMV7_DWARF_UUID=${IOS_DWARFDUMP_OUT[9]} + export RESULT_IOS_ARM64_DWARF_UUID=${IOS_DWARFDUMP_OUT[13]} + export RESULT_TVOS_ARM64_DWARF_UUID=${TVOS_DWARFDUMP_OUT[5]} + export RESULT_WATCHOS_ARMV7K_DWARF_UUID=${WATCHOS_DWARFDUMP_OUT[5]} + + echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 +} + +teardown() { + cd $BATS_TEST_DIRNAME +} + +myfunc() { + rome upload --concurrently --skip-local-cahe --cache-prefix travis > out.txt + } + +@test "rome uploads all artifacts (dynamic, yml)" { + echo "# $(run rome upload --concurrently --skip-local-cache --cache-prefix travis)" >&3 + run myfunc + echo "# $(ls)" >&3 + echo "# output of file: $(cat out.txt)" >&3 + [ 1 == 2 ] + + # Version file + [ -f "minio-buckets/rome/travis/Alamofire/.Alamofire.version-${ALAMOFIRE_VERSION}" ] + [ -f "rome-local-cache/travis/Alamofire/.Alamofire.version-${ALAMOFIRE_VERSION}" ] + [ -f "minio-buckets/rome/travis/Result/.Result.version-4.0.0" ] + [ -f "rome-local-cache/travis/Result/.Result.version-4.0.0" ] + + #save the local cache for later + + rm -rf ../_rome-local-cache_bkp + cp -R rome-local-cache/ ../_rome-local-cache_bkp + +} \ No newline at end of file diff --git a/integration-tests/engine.sh b/integration-tests/engine.sh new file mode 100755 index 0000000..0887442 --- /dev/null +++ b/integration-tests/engine.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +set -e + +ACTION="$1" + +STORAGE_DIR="server-cache" + +if [ "$ACTION" == "upload" ]; then + LOCAL_PATH="$2" + REMOTE_PATH="$3" + # POST request to upload file to remote path + # create directory structure if it doesn't exist yet + mkdir -p $STORAGE_DIR/$(dirname $REMOTE_PATH) + # fake the upload of a file by just copying binary to the storage directory + cp $LOCAL_PATH $STORAGE_DIR/$REMOTE_PATH + +elif [ "$ACTION" == "download" ]; then + REMOTE_PATH="$2" + OUTPUT_PATH="$3" + # fake download by just copying binary from the storage directory + cp $STORAGE_DIR/$REMOTE_PATH $OUTPUT_PATH + +elif [ "$ACTION" == "list" ]; then + REMOTE_PATH="$2" + # fake list command by just checking if file exists + if [ ! -f "$STORAGE_DIR/$REMOTE_PATH" ]; then + exit 1 + fi + +else + # unsupported command + exit 1 +fi From 9c9ebc288f2c4175347104252e3d597143914771 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 15:17:39 +0200 Subject: [PATCH 28/41] More documentation, error messages and fix typos --- src/Engine/Downloading.hs | 23 +++++++++++------------ src/Engine/Probing.hs | 4 ++-- src/Engine/Uploading.hs | 22 ++++++++++------------ 3 files changed, 23 insertions(+), 26 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index 61ee3d9..b5a1884 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -24,7 +24,7 @@ import qualified Turtle -- | Retrieves a Framework using the engine and unzip the contents getFrameworkFromEngine - :: FilePath -- ^ The engine file path + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -43,7 +43,7 @@ getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework -- | Retrieves a .version file using the engine getVersionFileFromEngine - :: FilePath + :: FilePath -- ^ The `FilePath` to the engine -> ProjectNameAndVersion -> ExceptT String @@ -61,9 +61,9 @@ getVersionFileFromEngine enginePath projectNameAndVersion = do versionFileRemotePath = remoteVersionFilePath projectNameAndVersion --- | Retrieves a bcsymbolmap with the engine +-- | Retrieves a bcsymbolmap using the engine getBcsymbolmapWithEngine - :: FilePath -- ^ The cache definition + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -88,7 +88,7 @@ getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framewor -- | Retrieves a dSYM using the engine getDSYMFromEngine - :: FilePath -- ^ The cache definition + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -109,7 +109,7 @@ getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ -- | Retrieves a bcsymbolmap using the engine and unzip the contents getAndUnzipBcsymbolmapWithEngine - :: FilePath -- ^ The engine path + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -135,7 +135,7 @@ getAndUnzipBcsymbolmapWithEngine enginePath reverseRomeMap fVersion@(FrameworkVe -- | Retrieves all the bcsymbolmap files using the engine and unzip the contents getAndUnzipBcsymbolmapsWithEngine' - :: FilePath -- ^ The cache definition + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -144,7 +144,6 @@ getAndUnzipBcsymbolmapsWithEngine' (ReaderT (CachePrefix, Bool) IO) () getAndUnzipBcsymbolmapsWithEngine' enginePath reverseRomeMap fVersion@(FrameworkVersion f@(Framework fwn _ fwps) _) platform - -- = undefined = when (platform `elem` fwps) $ do dwarfUUIDs <- withExceptT (const ErrorGettingDwarfUUIDs) @@ -174,7 +173,7 @@ getAndUnzipBcsymbolmapsWithEngine' enginePath reverseRomeMap fVersion@(Framework -- | Retrieves a Framework using the engine and unzip the contents getAndUnzipFrameworkWithEngine - :: FilePath -- ^ The path to the engine + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the Framework in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -198,7 +197,7 @@ getAndUnzipFrameworkWithEngine enginePath reverseRomeMap fVersion@(FrameworkVers -- | Retrieves a dSYM using the engine and unzip the contents getAndUnzipDSYMWithEngine - :: FilePath -- ^ The cache definition + :: FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve from a `FrameworkVersion` to the path of the dSYM in the cache -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the dSYM -> TargetPlatform -- ^ The `TargetPlatform` to limit the operation to @@ -213,7 +212,7 @@ getAndUnzipDSYMWithEngine enginePath reverseRomeMap fVersion@(FrameworkVersion f -- | Retrieves an artifact using the engine getArtifactFromEngine - :: FilePath -- ^ The engine file path + :: FilePath -- ^ The `FilePath` to the engine -> FilePath -- ^ The remote path -> String -- ^ A colloquial name for the artifact -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString @@ -233,7 +232,7 @@ getArtifactFromEngine enginePath remotePath artifactName = do -- | Downloads an artifact stored at a given path using the engine downloadBinaryWithEngine - :: FilePath + :: FilePath -- ^ The `FilePath` to the engine -> FilePath -> FilePath -> (ReaderT (Bool) IO) LBS.ByteString diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs index be3ec92..520fe47 100644 --- a/src/Engine/Probing.hs +++ b/src/Engine/Probing.hs @@ -27,7 +27,7 @@ probeEngineForFrameworks lCacheDir reverseRomeMap frameworkVersions frameworkVersions --- | Probes a `FilePath` to check if a `FrameworkVersion` exists for each `TargetPlatform` +-- | Probes the engine at `FilePath` to check if a `FrameworkVersion` exists for each `TargetPlatform` probeEngineForFramework :: MonadIO m => FilePath -- ^ The engine path. @@ -46,7 +46,7 @@ probeEngineForFramework lCacheDir reverseRomeMap frameworkVersion platforms (platforms `intersect` (_frameworkPlatforms . _framework $ frameworkVersion)) --- | Probes a `FilePath` to check if a `FrameworkVersion` exists for a given `TargetPlatform` +-- | Probes the engine at `FilePath` to check if a `FrameworkVersion` exists for a given `TargetPlatform` probeEngineForFrameworkOnPlatform :: MonadIO m => FilePath -- ^ The engine path. diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index bdfebb0..34f5c7a 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -10,9 +10,6 @@ import qualified Data.ByteString.Lazy as LBS import Data.Carthage.TargetPlatform import Data.Monoid ((<>)) import Data.Romefile (Framework (..)) -import qualified Data.Text as T -import qualified Network.AWS as AWS -import qualified Network.AWS.S3 as S3 import System.FilePath (()) import Types hiding (version) import Utils @@ -84,9 +81,9 @@ uploadBcsymbolmapToEngine dwarfUUID dwarfArchive enginePath reverseRomeMap (Fram remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version --- | Uploads a .version file to an engine +-- | Uploads a .version file using an engine uploadVersionFileToEngine' - :: FilePath -- ^ The engine definition. + :: FilePath -- ^ The engine path. -> LBS.ByteString -- ^ The contents of the .version file. -> ProjectNameAndVersion -- ^ The information used to derive the name and path for the .version file. -> ReaderT (CachePrefix, Bool) IO () @@ -104,10 +101,10 @@ uploadVersionFileToEngine' enginePath versionFileContent projectNameAndVersion = --- | Uploads an artifact to an engine +-- | Uploads an artifact using an engine uploadBinary :: MonadIO a - => FilePath + => FilePath -- ^ The engine path. -> LBS.ByteString -> FilePath -> FilePath @@ -117,12 +114,11 @@ uploadBinary enginePath binaryZip destinationPath objectName = (verbose) <- ask let cmd = Turtle.fromString $ enginePath liftIO $ saveBinaryToFile binaryZip destinationPath - sayLn -- TODO: probably remove this debugging log if not verbose, or maybe always? - $ "Executing script " + when verbose + $ sayLn + $ "Invoking engine " <> (show enginePath) <> " to upload " - <> objectName - <> " from: " <> destinationPath (exitCode) <- Turtle.proc cmd @@ -132,6 +128,8 @@ uploadBinary enginePath binaryZip destinationPath objectName = Turtle.ExitSuccess -> return () Turtle.ExitFailure n -> do sayLn - $ "Error: could not upload " + $ "Error " + <> (show n) + <> ": could not upload " <> destinationPath From 0a1c512b049ffcee646311633968c144db6c512b Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 15:55:29 +0200 Subject: [PATCH 29/41] Add dynamic frameworks engine yml integration test --- .../dynamic-frameworks-engine-yml.bats | 230 ++++++++++++++++-- integration-tests/engine.sh | 7 +- 2 files changed, 217 insertions(+), 20 deletions(-) diff --git a/integration-tests/dynamic-frameworks-engine-yml.bats b/integration-tests/dynamic-frameworks-engine-yml.bats index 4e5721c..204ce84 100644 --- a/integration-tests/dynamic-frameworks-engine-yml.bats +++ b/integration-tests/dynamic-frameworks-engine-yml.bats @@ -5,19 +5,20 @@ setup() { export ALAMOFIRE_VERSION="4.7.3" export RESULT_VERSION="4.0.0" - cd $BATS_TMPDIR - - rm -rf Rome-Tests - - mkdir Rome-Tests && cd Rome-Tests + rm -rf $BATS_TMPDIR/Rome-Tests + mkdir -p $BATS_TMPDIR/Rome-Tests + cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cd $BATS_TMPDIR/Rome-Tests if [ "$BATS_TEST_NUMBER" -eq 1 ]; then printf "github \"Alamofire/Alamofire\" == ${ALAMOFIRE_VERSION}\n" > Cartfile printf "github \"antitypical/Result\" == ${RESULT_VERSION}\n" >> Cartfile - # carthage bootstrap --cache-builds --no-use-binaries + # TODO: uncomment following line + #carthage bootstrap --cache-builds --no-use-binaries rm -rf ../_Carthage_build_bkp + # TODO: remove this line that was only for testing purposes cp -R /Users/balestrapatrick/Desktop/integration-test/ . cp -R Carthage/Build/ ../_Carthage_build_bkp @@ -62,32 +63,223 @@ EOF export RESULT_WATCHOS_ARMV7K_DWARF_UUID=${WATCHOS_DWARFDUMP_OUT[5]} echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 + } teardown() { cd $BATS_TEST_DIRNAME } -myfunc() { - rome upload --concurrently --skip-local-cahe --cache-prefix travis > out.txt - } +@test "rome uploads all artifacts with engine (dynamic, yml)" { -@test "rome uploads all artifacts (dynamic, yml)" { - echo "# $(run rome upload --concurrently --skip-local-cache --cache-prefix travis)" >&3 - run myfunc - echo "# $(ls)" >&3 - echo "# output of file: $(cat out.txt)" >&3 - [ 1 == 2 ] + # TODO: remove custom path and use `rome` + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis + + [ "$status" -eq 0 ] # Version file - [ -f "minio-buckets/rome/travis/Alamofire/.Alamofire.version-${ALAMOFIRE_VERSION}" ] + [ -f "server-cache/travis/Alamofire/.Alamofire.version-${ALAMOFIRE_VERSION}" ] [ -f "rome-local-cache/travis/Alamofire/.Alamofire.version-${ALAMOFIRE_VERSION}" ] - [ -f "minio-buckets/rome/travis/Result/.Result.version-4.0.0" ] - [ -f "rome-local-cache/travis/Result/.Result.version-4.0.0" ] + [ -f "server-cache/travis/Result/.Result.version-${RESULT_VERSION}" ] + [ -f "rome-local-cache/travis/Result/.Result.version-${RESULT_VERSION}" ] + + # macOS - No bitecode, No bcsymbolmap + + # macOS - Alamofire + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + + # macOS - Result + [ -f "server-cache/travis/Result/Mac/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/Mac/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/Mac/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/Mac/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + + # iOS - Alamofire + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${ALAMOFIRE_IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${ALAMOFIRE_IOS_ARM64_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${ALAMOFIRE_IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${ALAMOFIRE_IOS_ARM64_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] - #save the local cache for later + # iOS - Result + [ -f "server-cache/travis/Result/iOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/iOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/iOS/${RESULT_IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/iOS/${RESULT_IOS_ARM64_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/iOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/iOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/iOS/${RESULT_IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/iOS/${RESULT_IOS_ARM64_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + + # tvOS - Alamofire + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/${ALAMOFIRE_TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/${ALAMOFIRE_TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + # tvOS - Result + [ -f "server-cache/travis/Result/tvOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/tvOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/tvOS/${RESULT_TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/tvOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/tvOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/tvOS/${RESULT_TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + + # watchOS + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/${ALAMOFIRE_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${ALAMOFIRE_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/${ALAMOFIRE_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${ALAMOFIRE_VERSION}.zip" ] + + [ -f "server-cache/travis/Result/watchOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/watchOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "server-cache/travis/Result/watchOS/${RESULT_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/watchOS/Result.framework-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/watchOS/Result.framework.dSYM-${RESULT_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Result/watchOS/${RESULT_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${RESULT_VERSION}.zip" ] + + # save the server cache for later + rm -rf ../_server-cache_bkp + cp -R server-cache/ ../_server-cache_bkp + + # save the local cache for later rm -rf ../_rome-local-cache_bkp cp -R rome-local-cache/ ../_rome-local-cache_bkp +} + +@test "rome downloads all artifacts with engine skipping local cache (dynamic, yml)" { + + # restore server cache + if [ -d "../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../_server-cache_bkp server-cache/ + # echo "# $(ls)" >&3 + fi + + # restore local cache (even though it will be skipped, we want it to be there to simulate a real scenario) + if [ -d "../_server-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + # TODO: remove custom path and use `rome` + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --skip-local-cache --cache-prefix travis + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Alamofire.version" ] + [ -f "Carthage/Build/.Result.version" ] + + # macOS - No bitcode, No bcsymbolmap + # macOS - Alamofire + [ ! -d "Carthage/Build/Mac/Alamofire.framework" ] + [ ! -d "Carthage/Build/Mac/Alamofire.framework.dSYM" ] + + # macOS - Result + [ -d "Carthage/Build/Mac/Result.framework" ] + [ -d "Carthage/Build/Mac/Result.framework.dSYM" ] + + # iOS - Alamofire + [ -d "Carthage/Build/iOS/Alamofire.framework" ] + [ -d "Carthage/Build/iOS/Alamofire.framework.dSYM" ] + [ -e "Carthage/Build/iOS/${ALAMOFIRE_IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -e "Carthage/Build/iOS/${ALAMOFIRE_IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # iOS - Result + [ -d "Carthage/Build/iOS/Result.framework" ] + [ -d "Carthage/Build/iOS/Result.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${RESULT_IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${RESULT_IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS - Alamofire + [ -d "Carthage/Build/tvOS/Alamofire.framework" ] + [ -d "Carthage/Build/tvOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${ALAMOFIRE_TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS - Result + [ -d "Carthage/Build/tvOS/Result.framework" ] + [ -d "Carthage/Build/tvOS/Result.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${RESULT_TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS - Alamofire + [ -d "Carthage/Build/watchOS/Alamofire.framework" ] + [ -d "Carthage/Build/watchOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${ALAMOFIRE_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] + + # watchOS - Result + [ -d "Carthage/Build/watchOS/Result.framework" ] + [ -d "Carthage/Build/watchOS/Result.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${RESULT_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} + +@test "rome downloads all artifacts with engine from the local cache (dynamic, yml)" { + + # restore local cache + if [ -d "../_server-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + # TODO: remove custom path and use `rome` + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Alamofire.version" ] + [ -f "Carthage/Build/.Result.version" ] + + # macOS - No bitecode, No bcsymbolmap + # macOS - Alamofire + [ ! -d "Carthage/Build/Mac/Alamofire.framework" ] + [ ! -d "Carthage/Build/Mac/Alamofire.framework.dSYM" ] + + # macOS - Result + [ -d "Carthage/Build/Mac/Result.framework" ] + [ -d "Carthage/Build/Mac/Result.framework.dSYM" ] + + # iOS - Alamofire + [ -d "Carthage/Build/iOS/Alamofire.framework" ] + [ -d "Carthage/Build/iOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${ALAMOFIRE_IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${ALAMOFIRE_IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # iOS - Result + [ -d "Carthage/Build/iOS/Result.framework" ] + [ -d "Carthage/Build/iOS/Result.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${RESULT_IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${RESULT_IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS - Alamofire + [ -d "Carthage/Build/tvOS/Alamofire.framework" ] + [ -d "Carthage/Build/tvOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${ALAMOFIRE_TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS - Result + [ -d "Carthage/Build/tvOS/Result.framework" ] + [ -d "Carthage/Build/tvOS/Result.framework.dSYM" ] + [ -e "Carthage/Build/tvOS/${RESULT_TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS - Alamofire + [ -d "Carthage/Build/watchOS/Alamofire.framework" ] + [ -d "Carthage/Build/watchOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${ALAMOFIRE_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] + # watchOS - Result + [ -d "Carthage/Build/watchOS/Result.framework" ] + [ -d "Carthage/Build/watchOS/Result.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${RESULT_WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] } \ No newline at end of file diff --git a/integration-tests/engine.sh b/integration-tests/engine.sh index 0887442..a334717 100755 --- a/integration-tests/engine.sh +++ b/integration-tests/engine.sh @@ -18,8 +18,13 @@ if [ "$ACTION" == "upload" ]; then elif [ "$ACTION" == "download" ]; then REMOTE_PATH="$2" OUTPUT_PATH="$3" + # create directory structure if it doesn't exist yet + mkdir -p $(dirname $OUTPUT_PATH) # fake download by just copying binary from the storage directory - cp $STORAGE_DIR/$REMOTE_PATH $OUTPUT_PATH + # don't fail script if copy fails (some bcsymbolmap get requested wrongly at the moment, bug in Rome) + if [[ -f $STORAGE_DIR/$REMOTE_PATH ]]; then + cp $STORAGE_DIR/$REMOTE_PATH $OUTPUT_PATH + fi elif [ "$ACTION" == "list" ]; then REMOTE_PATH="$2" From a0944118eef8828f32754a779109622e489821b9 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 21:51:57 +0200 Subject: [PATCH 30/41] Improve engine logging messages --- src/Engine/Downloading.hs | 4 ++-- src/Engine/Uploading.hs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index b5a1884..a3bb9e1 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -241,8 +241,8 @@ downloadBinaryWithEngine enginePath objectRemotePath objectName = do let cmd = Turtle.fromString $ enginePath let sayFunc = if verbose then sayLnWithTime else sayLn when verbose - $ sayFunc - $ "Executing script" + $ sayLnWithTime + $ "Invoking engine " <> (show enginePath) <> " to download " <> objectName diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 34f5c7a..f346b32 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -115,7 +115,7 @@ uploadBinary enginePath binaryZip destinationPath objectName = let cmd = Turtle.fromString $ enginePath liftIO $ saveBinaryToFile binaryZip destinationPath when verbose - $ sayLn + $ sayLnWithTime $ "Invoking engine " <> (show enginePath) <> " to upload " From 9c583d203b6455ed469867c8bf988daaa3253894 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 23:00:37 +0200 Subject: [PATCH 31/41] Add engine integration tests to cover all cases (with local Carthage speedup) --- .../current-framework-engine-yml.bats | 280 ++++++++++++++++++ .../current-framework-named-engine-yml.bats | 224 ++++++++++++++ .../current-framework-named-yaml.bats | 4 +- integration-tests/current-framework-yaml.bats | 4 +- .../dynamic-frameworks-engine-yml.bats | 5 +- .../static-frameworks-engine-yml.bats | 199 +++++++++++++ 6 files changed, 713 insertions(+), 3 deletions(-) create mode 100644 integration-tests/current-framework-engine-yml.bats create mode 100644 integration-tests/current-framework-named-engine-yml.bats create mode 100644 integration-tests/static-frameworks-engine-yml.bats diff --git a/integration-tests/current-framework-engine-yml.bats b/integration-tests/current-framework-engine-yml.bats new file mode 100644 index 0000000..2bc94cb --- /dev/null +++ b/integration-tests/current-framework-engine-yml.bats @@ -0,0 +1,280 @@ +#!/usr/bin/env bats + +setup() { + + export FRAMEWORK_VERSION=4.8.2 + + rm -rf $BATS_TMPDIR/Rome-Tests + + mkdir -p $BATS_TMPDIR/Rome-Tests + + cp engine.sh $BATS_TMPDIR/Rome-Tests/ + + cd $BATS_TMPDIR/Rome-Tests + + git clone https://github.com/Alamofire/Alamofire.git + cd Alamofire + git checkout ${FRAMEWORK_VERSION} + + # cp ../engine.sh . + # cd .. + + if [ "$BATS_TEST_NUMBER" -eq 1 ]; then + + # TODO: uncomment following line + # carthage build --no-use-binaries --no-skip-current --cache-builds + mkdir -p Carthage/Build + cp -R /Users/balestrapatrick/Desktop/integration-test-alamofire/Alamofire/Carthage/Build/ Carthage/Build + + rm -rf ../../_Carthage_build_bkp + cp -R Carthage/Build/ ../../_Carthage_build_bkp + + else + mkdir -p Carthage/Build + cp -R ../../_Carthage_build_bkp/ Carthage/Build + fi + + cat >> Romefile << EOF +cache: + local: rome-local-cache + engine: ../engine.sh +ignoreMap: + - Alamofire: + - name: Alamofire + platforms: [Mac] +currentMap: + - Alamofire: + - name: Alamofire +EOF + + IOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/iOS/Alamofire.framework/Alamofire)) + TVOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/tvOS/Alamofire.framework/Alamofire)) + WATCHOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/watchOS/Alamofire.framework/Alamofire)) + + export IOS_ARMV7_DWARF_UUID=${IOS_DWARFDUMP_OUT[9]} + export IOS_ARM64_DWARF_UUID=${IOS_DWARFDUMP_OUT[13]} + export TVOS_ARM64_DWARF_UUID=${TVOS_DWARFDUMP_OUT[5]} + export WATCHOS_ARMV7K_DWARF_UUID=${WATCHOS_DWARFDUMP_OUT[5]} + + echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 + +} + +teardown() { + cd $BATS_TEST_DIRNAME +} + +@test "rome uploads all artifacts for current framework with engine (dynamic, yaml)" { + + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --no-skip-current + + [ "$status" -eq 0 ] + + # Version file + [ -f "server-cache/travis/Alamofire/.Alamofire.version-${FRAMEWORK_VERSION}" ] + [ -f "rome-local-cache/travis/Alamofire/.Alamofire.version-${FRAMEWORK_VERSION}" ] + + # macOS - No bitecode, No bcsymbolmap + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + + # iOS + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # tvOS + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # watchOS + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # save the server cache for later + rm -rf ../../_server-cache_bkp + cp -R server-cache/ ../../_server-cache_bkp + + # save the local cache for later + rm -rf ../../_rome-local-cache_bkp + cp -R rome-local-cache/ ../../_rome-local-cache_bkp +} + +@test "rome downloads all artifacts for current framework with engine skipping local cache (dynamic, yaml)" { + + # restore server cache + if [ -d "../../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../../_server-cache_bkp server-cache/ + fi + + # restore local cache (even though it will be skipped, we want it to be there to simulate a real scenario) + if [ -d "../../_rome-local-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Alamofire.version" ] + + # macOS - No bitcode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/Alamofire.framework" ] + [ ! -d "Carthage/Build/Mac/Alamofire.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/Alamofire.framework" ] + [ -d "Carthage/Build/iOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/Alamofire.framework" ] + [ -d "Carthage/Build/tvOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/Alamofire.framework" ] + [ -d "Carthage/Build/watchOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} + +@test "rome downloads all artifacts for current framework with engine from the local cache (dynamic, yaml)" { + + # restore local cache + if [ -d "../../_rome-local-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --no-skip-current + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Alamofire.version" ] + + # macOS - No bitecode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/Alamofire.framework" ] + [ ! -d "Carthage/Build/Mac/Alamofire.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/Alamofire.framework" ] + [ -d "Carthage/Build/iOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/Alamofire.framework" ] + [ -d "Carthage/Build/tvOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/Alamofire.framework" ] + [ -d "Carthage/Build/watchOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} + +@test "rome uploads named artifacts for current framework with engine (dynamic, yaml)" { + + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --no-skip-current Alamofire + + [ "$status" -eq 0 ] + + # Version file + [ -f "server-cache/travis/Alamofire/.Alamofire.version-${FRAMEWORK_VERSION}" ] + [ -f "rome-local-cache/travis/Alamofire/.Alamofire.version-${FRAMEWORK_VERSION}" ] + + # macOS - No bitecode, No bcsymbolmap + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "server-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Alamofire/Mac/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + + # iOS + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # tvOS + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # watchOS + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/Alamofire/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/Alamofire.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/Alamofire/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # save the local cache for later + rm -rf ../../_rome-local-cache_bkp + cp -R rome-local-cache/ ../../_rome-local-cache_bkp +} + +@test "rome downloads named artifacts for current framework with engine skipping local cache (dynamic, yaml)" { + + # restore server cache + if [ -d "../../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../../_server-cache_bkp server-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current Alamofire + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Alamofire.version" ] + + # macOS - No bitcode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/Alamofire.framework" ] + [ ! -d "Carthage/Build/Mac/Alamofire.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/Alamofire.framework" ] + [ -d "Carthage/Build/iOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/Alamofire.framework" ] + [ -d "Carthage/Build/tvOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/Alamofire.framework" ] + [ -d "Carthage/Build/watchOS/Alamofire.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} \ No newline at end of file diff --git a/integration-tests/current-framework-named-engine-yml.bats b/integration-tests/current-framework-named-engine-yml.bats new file mode 100644 index 0000000..f8d68c3 --- /dev/null +++ b/integration-tests/current-framework-named-engine-yml.bats @@ -0,0 +1,224 @@ +#!/usr/bin/env bats + +setup() { + + export FRAMEWORK_VERSION=0.4.0 + export FRAMEWORK_REPO_NAME=swift-tagged + export FRAMEWORK_ARTIFACT_NAME=Tagged + + rm -rf $BATS_TMPDIR/Rome-Tests + + mkdir -p $BATS_TMPDIR/Rome-Tests + + cp engine.sh $BATS_TMPDIR/Rome-Tests/ + + cd $BATS_TMPDIR/Rome-Tests + + git clone https://github.com/pointfreeco/swift-tagged.git + cd swift-tagged + git checkout ${FRAMEWORK_VERSION} + + if [ "$BATS_TEST_NUMBER" -eq 1 ]; then + + # carthage build --no-use-binaries --no-skip-current --cache-builds + mkdir -p Carthage/Build + cp -R /Users/balestrapatrick/Desktop/integration-named/swift-tagged/Carthage/Build/ Carthage/Build + + rm -rf ../../_Carthage_build_bkp + cp -R Carthage/Build/ ../../_Carthage_build_bkp + + else + mkdir -p Carthage/Build + cp -R ../../_Carthage_build_bkp/ Carthage/Build + fi + + cat >> Romefile << EOF +cache: + local: rome-local-cache + engine: ../engine.sh +ignoreMap: + - swift-tagged: + - name: Tagged + platforms: [Mac] +currentMap: + - swift-tagged: + - name: Tagged +EOF + + IOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework/${FRAMEWORK_ARTIFACT_NAME})) + TVOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework/${FRAMEWORK_ARTIFACT_NAME})) + WATCHOS_DWARFDUMP_OUT=($(dwarfdump -u Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework/${FRAMEWORK_ARTIFACT_NAME})) + + export IOS_ARMV7_DWARF_UUID=${IOS_DWARFDUMP_OUT[9]} + export IOS_ARM64_DWARF_UUID=${IOS_DWARFDUMP_OUT[13]} + export TVOS_ARM64_DWARF_UUID=${TVOS_DWARFDUMP_OUT[5]} + export WATCHOS_ARMV7K_DWARF_UUID=${WATCHOS_DWARFDUMP_OUT[5]} + + echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 + +} + +teardown() { + cd $BATS_TEST_DIRNAME +} + +@test "rome uploads all named artifacts for current framework with engine (dynamic, yaml)" { + echo "# $(pwd)" >&3 + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} + [ "$status" -eq 0 ] + + # Version file + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/.${FRAMEWORK_REPO_NAME}.version-${FRAMEWORK_VERSION}" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/.${FRAMEWORK_REPO_NAME}.version-${FRAMEWORK_VERSION}" ] + + # macOS - No bitecode, No bcsymbolmap + [ ! -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + + # iOS + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # tvOS + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # watchOS + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "server-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM-${FRAMEWORK_VERSION}.zip" ] + [ -f "rome-local-cache/travis/${FRAMEWORK_REPO_NAME}/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap-${FRAMEWORK_VERSION}.zip" ] + + # save the server cache for later + rm -rf ../../_server-cache_bkp + cp -R server-cache/ ../../_server-cache_bkp + + # save the local cache for later + rm -rf ../../_rome-local-cache_bkp + cp -R rome-local-cache/ ../../_rome-local-cache_bkp +} + +@test "rome downloads all named artifacts for current framework with engine skipping local cache (dynamic, yaml)" { + + # restore server cache + if [ -d "../../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../../_server-cache_bkp server-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.${FRAMEWORK_REPO_NAME}.version" ] + + # macOS - No bitcode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} + +@test "rome downloads all named artifacts for current framework with engine from the local cache (dynamic, yaml)" { + + # restore local cache + if [ -d "../../_rome-local-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.${FRAMEWORK_REPO_NAME}.version" ] + + # macOS - No bitecode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} + +@test "rome downloads named artifacts for current framework with engine skipping local cache (dynamic, yaml)" { + + # restore server cache + if [ -d "../../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../../_server-cache_bkp server-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.${FRAMEWORK_REPO_NAME}.version" ] + + # macOS - No bitcode, No bcsymbolmap + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ ! -d "Carthage/Build/Mac/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + + # iOS + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/iOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/iOS/${IOS_ARMV7_DWARF_UUID}.bcsymbolmap" ] + [ -f "Carthage/Build/iOS/${IOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # tvOS + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/tvOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/tvOS/${TVOS_ARM64_DWARF_UUID}.bcsymbolmap" ] + + # watchOS + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework" ] + [ -d "Carthage/Build/watchOS/${FRAMEWORK_ARTIFACT_NAME}.framework.dSYM" ] + [ -f "Carthage/Build/watchOS/${WATCHOS_ARMV7K_DWARF_UUID}.bcsymbolmap" ] +} diff --git a/integration-tests/current-framework-named-yaml.bats b/integration-tests/current-framework-named-yaml.bats index a7ae7d7..1f763aa 100644 --- a/integration-tests/current-framework-named-yaml.bats +++ b/integration-tests/current-framework-named-yaml.bats @@ -18,7 +18,9 @@ setup() { if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - carthage build --no-use-binaries --no-skip-current --cache-builds + # carthage build --no-use-binaries --no-skip-current --cache-builds + mkdir -p Carthage/Build + cp -R /Users/balestrapatrick/Desktop/integration-named/swift-tagged/Carthage/Build/ Carthage/Build rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp diff --git a/integration-tests/current-framework-yaml.bats b/integration-tests/current-framework-yaml.bats index 8fd3384..94083f8 100644 --- a/integration-tests/current-framework-yaml.bats +++ b/integration-tests/current-framework-yaml.bats @@ -16,7 +16,9 @@ setup() { if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - carthage build --no-use-binaries --no-skip-current --cache-builds + # carthage build --no-use-binaries --no-skip-current --cache-builds + mkdir -p Carthage/Build + cp -R /Users/balestrapatrick/Desktop/integration-test-alamofire/Alamofire/Carthage/Build/ Carthage/Build rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp diff --git a/integration-tests/dynamic-frameworks-engine-yml.bats b/integration-tests/dynamic-frameworks-engine-yml.bats index 204ce84..b60fe80 100644 --- a/integration-tests/dynamic-frameworks-engine-yml.bats +++ b/integration-tests/dynamic-frameworks-engine-yml.bats @@ -6,8 +6,11 @@ setup() { export RESULT_VERSION="4.0.0" rm -rf $BATS_TMPDIR/Rome-Tests + mkdir -p $BATS_TMPDIR/Rome-Tests + cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cd $BATS_TMPDIR/Rome-Tests if [ "$BATS_TEST_NUMBER" -eq 1 ]; then @@ -66,6 +69,7 @@ EOF } + teardown() { cd $BATS_TEST_DIRNAME } @@ -163,7 +167,6 @@ teardown() { if [ -d "../_server-cache_bkp" ]; then echo "# Server cache restored" >&3 cp -R ../_server-cache_bkp server-cache/ - # echo "# $(ls)" >&3 fi # restore local cache (even though it will be skipped, we want it to be there to simulate a real scenario) diff --git a/integration-tests/static-frameworks-engine-yml.bats b/integration-tests/static-frameworks-engine-yml.bats new file mode 100644 index 0000000..3744285 --- /dev/null +++ b/integration-tests/static-frameworks-engine-yml.bats @@ -0,0 +1,199 @@ +#!/usr/bin/env bats + +setup() { + + export STATICFIRE_COMMIT="ada9256edf4b6dfb63bd1f6d13441b1ab7fd6ffd" + export RESULT_VERSION="4.1.0" + + rm -rf $BATS_TMPDIR/Rome-Tests + + mkdir -p $BATS_TMPDIR/Rome-Tests + + cp engine.sh $BATS_TMPDIR/Rome-Tests/ + + cd $BATS_TMPDIR/Rome-Tests + + if [ "$BATS_TEST_NUMBER" -eq 1 ]; then + printf "github \"blender/Staticfire\" \"master\"\n" > Cartfile + printf "github \"antitypical/Result\" == ${RESULT_VERSION}\n" >> Cartfile + + # carthage bootstrap --cache-builds --no-use-binaries + + rm -rf ../_Carthage_build_bkp + # TODO: remove this line that was only for testing purposes + cp -R /Users/balestrapatrick/Desktop/integration-test-static/ . + cp -R Carthage/Build/ ../_Carthage_build_bkp + + rm -f ../_Cartfile_bkp + rm -f ../_Cartfile.resolved_bkp + cp Cartfile ../_Cartfile_bkp + cp Cartfile.resolved ../_Cartfile.resolved_bkp + + else + mkdir -p Carthage/Build + cp -R ../_Carthage_build_bkp/ Carthage/Build + cp ../_Cartfile_bkp Cartfile + cp ../_Cartfile.resolved_bkp Cartfile.resolved + fi + + cat >> ../Romefile.yaml << EOF +cache: + local: rome-local-cache + engine: engine.sh +repositoryMap: +- Staticfire: + - name: Alamofire + type: static +ignoreMap: +- Staticfire: + - name: Alamofire + type: static + platforms: [Mac] +- Result: + - name: Result + platforms: [iOS,Mac,tvOS,watchOS] +EOF + + # mkdir -p server-cache + + echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 + +} + +teardown() { + cd $BATS_TEST_DIRNAME +} + + +@test "rome uploads all artifacts with engine (static, yml)" { + + # No dSYMs nor bcsymbolmaps for Static Frameworks + + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --romefile ../Romefile.yaml + + [ "$status" -eq 0 ] + + # Version file + [ -f "server-cache/travis/Staticfire/.Staticfire.version-${STATICFIRE_COMMIT}" ] + [ -f "rome-local-cache/travis/Staticfire/.Staticfire.version-${STATICFIRE_COMMIT}" ] + [ ! -f "server-cache/travis/Result/.Result.version-${RESULT_VERSION}" ] + [ ! -f "rome-local-cache/travis/Result/.Result.version-${RESULT_VERSION}" ] + + # macOS - Staticfire + [ ! -f "server-cache/travis/Staticfire/Mac/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + [ ! -f "rome-local-cache/travis/Staticfire/Mac/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + # macOS - Result + [ ! -f "server-cache/travis/Result/Mac/Result.framework-${RESULT_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Result/Mac/Result.framework-${RESULT_VERSION}.zip" ] + + # iOS - Staticfire + [ -f "server-cache/travis/Staticfire/iOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + [ -f "rome-local-cache/travis/Staticfire/iOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + # iOS - Result + [ ! -f "server-cache/travis/Result/iOS/Result.framework-${RESULT_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Result/iOS/Result.framework-${RESULT_VERSION}.zip" ] + + # tvOS - Staticfire + [ -f "server-cache/travis/Staticfire/tvOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + [ -f "rome-local-cache/travis/Staticfire/tvOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + # tvOS - Result + [ ! -f "server-cache/travis/Result/tvOS/Result.framework-${RESULT_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Result/tvOS/Result.framework-${RESULT_VERSION}.zip" ] + + # watchOS - Staticfire + [ -f "server-cache/travis/Staticfire/watchOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + [ -f "rome-local-cache/travis/Staticfire/watchOS/Alamofire.framework-static-${STATICFIRE_COMMIT}.zip" ] + # watchOS - Result + [ ! -f "server-cache/travis/Result/watchOS/Result.framework-${RESULT_VERSION}.zip" ] + [ ! -f "rome-local-cache/travis/Result/watchOS/Result.framework-${RESULT_VERSION}.zip" ] + + # save the server cache for later + rm -rf ../_server-cache_bkp + cp -R server-cache/ ../_server-cache_bkp + + # save the local cache for later + rm -rf ../_rome-local-cache_bkp + cp -R rome-local-cache/ ../_rome-local-cache_bkp +} + +@test "rome downloads all artifacts with engine skipping local cache (static, yml)" { + + # No dSYMs nor bcsymbolmaps for Static Frameworks + + # restore server cache + if [ -d "../_server-cache_bkp" ]; then + echo "# Server cache restored" >&3 + cp -R ../_server-cache_bkp server-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --romefile ../Romefile.yaml + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Staticfire.version" ] + [ ! -f "Carthage/Build/.Result.version" ] + + + # macOS - Staticfire + [ ! -d "Carthage/Build/Mac/Static/Alamofire.framework" ] + # macOS - Result + [ ! -d "Carthage/Build/Mac/Result.framework" ] + + # iOS - Staticfire + [ -d "Carthage/Build/iOS/Static/Alamofire.framework" ] + # iOS - Result + [ ! -d "Carthage/Build/iOS/Result.framework" ] + + # tvOS - Staticfire + [ -d "Carthage/Build/tvOS/Static/Alamofire.framework" ] + # tvOS - Result + [ ! -d "Carthage/Build/tvOS/Result.framework" ] + + # watchOS - Staticfire + [ -d "Carthage/Build/watchOS/Static/Alamofire.framework" ] + # watchOS - Result + [ ! -d "Carthage/Build/watchOS/Result.framework" ] + +} + +@test "rome downloads all artifacts with engine from the local cache (static, yml)" { + + # No dSYMs nor bcsymbolmaps for Static Frameworks + + # restore local cache + if [ -d "../_rome-local-cache_bkp" ]; then + echo "# Local cache restored" >&3 + cp -R ../_rome-local-cache_bkp rome-local-cache/ + fi + + rm -rf Carthage/Build + run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --romefile ../Romefile.yaml + + [ "$status" -eq 0 ] + + # Version file + [ -f "Carthage/Build/.Staticfire.version" ] + + # macOS - No bitecode, No bcsymbolmap + # macOS - Staticfire + [ ! -d "Carthage/Build/Mac/Static/Alamofire.framework" ] + # macoS - Result + [ ! -d "Carthage/Build/Mac/Result.framework" ] + + # iOS - Staticfire + [ -d "Carthage/Build/iOS/Static/Alamofire.framework" ] + # iOS - Result + [ ! -d "Carthage/Build/iOS/Result.framework" ] + + # tvOS - Staticfire + [ -d "Carthage/Build/tvOS/Static/Alamofire.framework" ] + # tvOS - Result + [ ! -d "Carthage/Build/tvOS/Result.framework" ] + + # watchOS - Staticfire + [ -d "Carthage/Build/watchOS/Static/Alamofire.framework" ] + # watchOS - Result + [ ! -d "Carthage/Build/watchOS/Result.framework" ] +} From de4d964544ddf78f495208cee3d3513cdecb9eba Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 23:22:51 +0200 Subject: [PATCH 32/41] Clean up integration tests and add new ones to travis configuration --- .travis.yml | 8 ++++++-- .../current-framework-engine-yml.bats | 18 ++++++------------ .../current-framework-named-engine-yml.bats | 15 +++++++-------- ...l.bats => current-framework-named-yml.bats} | 0 ...rk-yaml.bats => current-framework-yml.bats} | 0 .../dynamic-frameworks-engine-yml.bats | 14 ++++---------- .../static-frameworks-engine-yml.bats | 12 ++++-------- 7 files changed, 27 insertions(+), 40 deletions(-) rename integration-tests/{current-framework-named-yaml.bats => current-framework-named-yml.bats} (100%) rename integration-tests/{current-framework-yaml.bats => current-framework-yml.bats} (100%) diff --git a/.travis.yml b/.travis.yml index f13bf69..f8f554e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -46,6 +46,7 @@ jobs: - stack $ARGS install - travis_wait 60 bats integration-tests/dynamic-frameworks-ini.bats - travis_wait 60 bats integration-tests/dynamic-frameworks-yml.bats + - travis_wait 60 bats integration-tests/dynamic-frameworks-engine-yml.bats - stage: "Build" name: "Build Rome & Test Current Frameworks" before_install: @@ -80,8 +81,10 @@ jobs: - stack $ARGS build -j 2 - stack $ARGS sdist - stack $ARGS install - - travis_wait 60 bats integration-tests/current-framework-yaml.bats - - travis_wait 60 bats integration-tests/current-framework-named-yaml.bats + - travis_wait 60 bats integration-tests/current-framework-yml.bats + - travis_wait 60 bats integration-tests/current-framework-engine-yml.bats + - travis_wait 60 bats integration-tests/current-framework-named-yml.bats + - travis_wait 60 bats integration-tests/current-framework-named-engine-yml.bats - stage: "Build" name: "Build Rome & Test Static Frameworks" before_install: @@ -118,6 +121,7 @@ jobs: - stack $ARGS install - travis_wait 60 bats integration-tests/static-frameworks-ini.bats - travis_wait 60 bats integration-tests/static-frameworks-yml.bats + - travis_wait 60 bats integration-tests/static-frameworks-engine-yml.bats env: - ARGS="--resolver=lts-13.10" diff --git a/integration-tests/current-framework-engine-yml.bats b/integration-tests/current-framework-engine-yml.bats index 2bc94cb..56cb114 100644 --- a/integration-tests/current-framework-engine-yml.bats +++ b/integration-tests/current-framework-engine-yml.bats @@ -16,15 +16,9 @@ setup() { cd Alamofire git checkout ${FRAMEWORK_VERSION} - # cp ../engine.sh . - # cd .. - if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - # TODO: uncomment following line - # carthage build --no-use-binaries --no-skip-current --cache-builds - mkdir -p Carthage/Build - cp -R /Users/balestrapatrick/Desktop/integration-test-alamofire/Alamofire/Carthage/Build/ Carthage/Build + carthage build --no-use-binaries --no-skip-current --cache-builds rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp @@ -66,7 +60,7 @@ teardown() { @test "rome uploads all artifacts for current framework with engine (dynamic, yaml)" { - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --no-skip-current + run rome upload --concurrently --cache-prefix travis --no-skip-current [ "$status" -eq 0 ] @@ -130,7 +124,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current + run rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current [ "$status" -eq 0 ] @@ -167,7 +161,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --no-skip-current + run rome download --concurrently --cache-prefix travis --no-skip-current [ "$status" -eq 0 ] @@ -197,7 +191,7 @@ teardown() { @test "rome uploads named artifacts for current framework with engine (dynamic, yaml)" { - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --no-skip-current Alamofire + run rome upload --concurrently --cache-prefix travis --no-skip-current Alamofire [ "$status" -eq 0 ] @@ -251,7 +245,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current Alamofire + run rome download --concurrently --cache-prefix travis --skip-local-cache --no-skip-current Alamofire [ "$status" -eq 0 ] diff --git a/integration-tests/current-framework-named-engine-yml.bats b/integration-tests/current-framework-named-engine-yml.bats index f8d68c3..3658dbd 100644 --- a/integration-tests/current-framework-named-engine-yml.bats +++ b/integration-tests/current-framework-named-engine-yml.bats @@ -20,9 +20,7 @@ setup() { if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - # carthage build --no-use-binaries --no-skip-current --cache-builds - mkdir -p Carthage/Build - cp -R /Users/balestrapatrick/Desktop/integration-named/swift-tagged/Carthage/Build/ Carthage/Build + carthage build --no-use-binaries --no-skip-current --cache-builds rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp @@ -63,8 +61,9 @@ teardown() { } @test "rome uploads all named artifacts for current framework with engine (dynamic, yaml)" { - echo "# $(pwd)" >&3 - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} + + run rome upload --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} + [ "$status" -eq 0 ] # Version file @@ -121,7 +120,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} + run rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} [ "$status" -eq 0 ] @@ -158,7 +157,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} + run rome download --concurrently --cache-prefix travis ${FRAMEWORK_REPO_NAME} [ "$status" -eq 0 ] @@ -195,7 +194,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} + run rome download --concurrently --cache-prefix travis --skip-local-cache ${FRAMEWORK_REPO_NAME} [ "$status" -eq 0 ] diff --git a/integration-tests/current-framework-named-yaml.bats b/integration-tests/current-framework-named-yml.bats similarity index 100% rename from integration-tests/current-framework-named-yaml.bats rename to integration-tests/current-framework-named-yml.bats diff --git a/integration-tests/current-framework-yaml.bats b/integration-tests/current-framework-yml.bats similarity index 100% rename from integration-tests/current-framework-yaml.bats rename to integration-tests/current-framework-yml.bats diff --git a/integration-tests/dynamic-frameworks-engine-yml.bats b/integration-tests/dynamic-frameworks-engine-yml.bats index b60fe80..43808b3 100644 --- a/integration-tests/dynamic-frameworks-engine-yml.bats +++ b/integration-tests/dynamic-frameworks-engine-yml.bats @@ -17,12 +17,9 @@ setup() { printf "github \"Alamofire/Alamofire\" == ${ALAMOFIRE_VERSION}\n" > Cartfile printf "github \"antitypical/Result\" == ${RESULT_VERSION}\n" >> Cartfile - # TODO: uncomment following line - #carthage bootstrap --cache-builds --no-use-binaries + carthage bootstrap --cache-builds --no-use-binaries rm -rf ../_Carthage_build_bkp - # TODO: remove this line that was only for testing purposes - cp -R /Users/balestrapatrick/Desktop/integration-test/ . cp -R Carthage/Build/ ../_Carthage_build_bkp rm -f ../_Cartfile_bkp @@ -76,8 +73,7 @@ teardown() { @test "rome uploads all artifacts with engine (dynamic, yml)" { - # TODO: remove custom path and use `rome` - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis + run rome upload --concurrently --cache-prefix travis [ "$status" -eq 0 ] @@ -176,8 +172,7 @@ teardown() { fi rm -rf Carthage/Build - # TODO: remove custom path and use `rome` - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --skip-local-cache --cache-prefix travis + run rome download --concurrently --skip-local-cache --cache-prefix travis [ "$status" -eq 0 ] @@ -236,8 +231,7 @@ teardown() { fi rm -rf Carthage/Build - # TODO: remove custom path and use `rome` - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis + run rome download --concurrently --cache-prefix travis [ "$status" -eq 0 ] diff --git a/integration-tests/static-frameworks-engine-yml.bats b/integration-tests/static-frameworks-engine-yml.bats index 3744285..fe6ac40 100644 --- a/integration-tests/static-frameworks-engine-yml.bats +++ b/integration-tests/static-frameworks-engine-yml.bats @@ -17,11 +17,9 @@ setup() { printf "github \"blender/Staticfire\" \"master\"\n" > Cartfile printf "github \"antitypical/Result\" == ${RESULT_VERSION}\n" >> Cartfile - # carthage bootstrap --cache-builds --no-use-binaries + carthage bootstrap --cache-builds --no-use-binaries rm -rf ../_Carthage_build_bkp - # TODO: remove this line that was only for testing purposes - cp -R /Users/balestrapatrick/Desktop/integration-test-static/ . cp -R Carthage/Build/ ../_Carthage_build_bkp rm -f ../_Cartfile_bkp @@ -53,8 +51,6 @@ ignoreMap: - name: Result platforms: [iOS,Mac,tvOS,watchOS] EOF - - # mkdir -p server-cache echo "# BATS_TMPDIR: ${BATS_TMPDIR}" >&3 @@ -69,7 +65,7 @@ teardown() { # No dSYMs nor bcsymbolmaps for Static Frameworks - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome upload --concurrently --cache-prefix travis --romefile ../Romefile.yaml + run rome upload --concurrently --cache-prefix travis --romefile ../Romefile.yaml [ "$status" -eq 0 ] @@ -127,7 +123,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --skip-local-cache --romefile ../Romefile.yaml + run rome download --concurrently --cache-prefix travis --skip-local-cache --romefile ../Romefile.yaml [ "$status" -eq 0 ] @@ -169,7 +165,7 @@ teardown() { fi rm -rf Carthage/Build - run /Users/balestrapatrick/GitHub/Rome/.stack-work/install/x86_64-osx/lts-13.10/8.6.3/bin/rome download --concurrently --cache-prefix travis --romefile ../Romefile.yaml + run rome download --concurrently --cache-prefix travis --romefile ../Romefile.yaml [ "$status" -eq 0 ] From abe94fe9de973c34cb1574552b9bba6164b52d72 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 23:43:51 +0200 Subject: [PATCH 33/41] Revert useless changes to existing integration tests --- integration-tests/current-framework-named-yml.bats | 4 +--- integration-tests/current-framework-yml.bats | 4 +--- 2 files changed, 2 insertions(+), 6 deletions(-) diff --git a/integration-tests/current-framework-named-yml.bats b/integration-tests/current-framework-named-yml.bats index 1f763aa..a7ae7d7 100644 --- a/integration-tests/current-framework-named-yml.bats +++ b/integration-tests/current-framework-named-yml.bats @@ -18,9 +18,7 @@ setup() { if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - # carthage build --no-use-binaries --no-skip-current --cache-builds - mkdir -p Carthage/Build - cp -R /Users/balestrapatrick/Desktop/integration-named/swift-tagged/Carthage/Build/ Carthage/Build + carthage build --no-use-binaries --no-skip-current --cache-builds rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp diff --git a/integration-tests/current-framework-yml.bats b/integration-tests/current-framework-yml.bats index 94083f8..8fd3384 100644 --- a/integration-tests/current-framework-yml.bats +++ b/integration-tests/current-framework-yml.bats @@ -16,9 +16,7 @@ setup() { if [ "$BATS_TEST_NUMBER" -eq 1 ]; then - # carthage build --no-use-binaries --no-skip-current --cache-builds - mkdir -p Carthage/Build - cp -R /Users/balestrapatrick/Desktop/integration-test-alamofire/Alamofire/Carthage/Build/ Carthage/Build + carthage build --no-use-binaries --no-skip-current --cache-builds rm -rf ../../_Carthage_build_bkp cp -R Carthage/Build/ ../../_Carthage_build_bkp From ba34d992ee11e99fbdf418622cf8188851f17568 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Thu, 16 May 2019 23:51:07 +0200 Subject: [PATCH 34/41] More documentation and style fixes --- src/Engine/Probing.hs | 8 ++++---- src/Engine/Uploading.hs | 11 ++++++----- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs index 520fe47..f43c918 100644 --- a/src/Engine/Probing.hs +++ b/src/Engine/Probing.hs @@ -14,7 +14,7 @@ import qualified Turtle -- | Probes a `FilePath` to check if each `FrameworkVersion` exists for each `TargetPlatform` probeEngineForFrameworks :: MonadIO m - => FilePath -- ^ The engine path. + => FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> [FrameworkVersion] -- ^ A list of `FrameworkVersion` to probe for. -> [TargetPlatform] -- ^ A list target platforms restricting the scope of this action. @@ -30,7 +30,7 @@ probeEngineForFrameworks lCacheDir reverseRomeMap frameworkVersions -- | Probes the engine at `FilePath` to check if a `FrameworkVersion` exists for each `TargetPlatform` probeEngineForFramework :: MonadIO m - => FilePath -- ^ The engine path. + => FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` to probe for. -> [TargetPlatform] -- ^ A list target platforms restricting the scope of this action. @@ -49,7 +49,7 @@ probeEngineForFramework lCacheDir reverseRomeMap frameworkVersion platforms -- | Probes the engine at `FilePath` to check if a `FrameworkVersion` exists for a given `TargetPlatform` probeEngineForFrameworkOnPlatform :: MonadIO m - => FilePath -- ^ The engine path. + => FilePath -- ^ The `FilePath` to the engine -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` to probe for. -> TargetPlatform -- ^ A target platforms restricting the scope of this action. @@ -62,7 +62,7 @@ probeEngineForFrameworkOnPlatform enginePath reverseRomeMap (FrameworkVersion fw ["list", (Turtle.fromString remoteFrameworkUploadPath)] (return $ Turtle.unsafeTextToLine "") case exitCode of - -- If script exits with success, we assume the framework exists. + -- If engine exits with success, we assume the framework exists. Turtle.ExitSuccess -> return (PlatformAvailability platform True) Turtle.ExitFailure n -> return (PlatformAvailability platform False) where diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index f346b32..2599f4c 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -20,7 +20,7 @@ import qualified Turtle -- | Uploads a Framework `Zip.Archive` to an engine. uploadFrameworkToEngine :: Zip.Archive -- ^ The `Zip.Archive` of the Framework. - -> FilePath -- ^ The engine definition. + -> FilePath -- ^ The `FilePath` to the engine. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework. -> TargetPlatform -- ^ A `TargetPlatform`s restricting the scope of this action. @@ -42,7 +42,7 @@ uploadFrameworkToEngine frameworkArchive enginePath reverseRomeMap (FrameworkVer -- | Uploads a dSYM `Zip.Archive` to an engine. uploadDsymToEngine :: Zip.Archive -- ^ The `Zip.Archive` of the dSYM. - -> FilePath -- ^ The engine definition. + -> FilePath -- ^ The `FilePath` to the engine. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. @@ -63,7 +63,7 @@ uploadDsymToEngine dSYMArchive enginePath reverseRomeMap (FrameworkVersion f@(Fr uploadBcsymbolmapToEngine :: DwarfUUID -- ^ The UUID of the bcsymbolmap -> Zip.Archive -- ^ The `Zip.Archive` of the dSYM. - -> FilePath -- ^ The engine definition. + -> FilePath -- ^ The `FilePath` to the engine. -> InvertedRepositoryMap -- ^ The map used to resolve `FrameworkName`s to `GitRepoName`s. -> FrameworkVersion -- ^ The `FrameworkVersion` identifying the Framework and the dSYM. -> TargetPlatform -- ^ A `TargetPlatform` restricting the scope of this action. @@ -81,9 +81,10 @@ uploadBcsymbolmapToEngine dwarfUUID dwarfArchive enginePath reverseRomeMap (Fram remoteBcsymbolmapPath dwarfUUID platform reverseRomeMap f version + -- | Uploads a .version file using an engine uploadVersionFileToEngine' - :: FilePath -- ^ The engine path. + :: FilePath -- ^ The `FilePath` to the engine. -> LBS.ByteString -- ^ The contents of the .version file. -> ProjectNameAndVersion -- ^ The information used to derive the name and path for the .version file. -> ReaderT (CachePrefix, Bool) IO () @@ -104,7 +105,7 @@ uploadVersionFileToEngine' enginePath versionFileContent projectNameAndVersion = -- | Uploads an artifact using an engine uploadBinary :: MonadIO a - => FilePath -- ^ The engine path. + => FilePath -- ^ The `FilePath` to the engine. -> LBS.ByteString -> FilePath -> FilePath From 8b11049a823c4b83e8ff86d90e614592304faa49 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Fri, 17 May 2019 10:27:34 +0200 Subject: [PATCH 35/41] Address PR comments and hlint warnings --- src/Engine/Downloading.hs | 22 +++++++++++----------- src/Engine/Probing.hs | 8 ++++---- src/Engine/Uploading.hs | 25 ++++++++++++------------- src/Lib.hs | 9 +-------- 4 files changed, 28 insertions(+), 36 deletions(-) diff --git a/src/Engine/Downloading.hs b/src/Engine/Downloading.hs index a3bb9e1..ce1373b 100644 --- a/src/Engine/Downloading.hs +++ b/src/Engine/Downloading.hs @@ -34,7 +34,7 @@ getFrameworkFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework (CachePrefix cachePrefix, verbose) <- ask let frameworkLocalPath = cachePrefix remoteFrameworkUploadPath mapExceptT - (withReaderT (const (verbose))) + (withReaderT (const verbose)) (getArtifactFromEngine enginePath frameworkLocalPath fwn ) where @@ -52,7 +52,7 @@ getVersionFileFromEngine getVersionFileFromEngine enginePath projectNameAndVersion = do (CachePrefix prefix, verbose) <- ask let finalVersionFileRemotePath = prefix versionFileRemotePath - mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine + mapExceptT (withReaderT (const verbose)) $ getArtifactFromEngine enginePath finalVersionFileRemotePath versionFileName @@ -76,7 +76,7 @@ getBcsymbolmapWithEngine enginePath reverseRomeMap (FrameworkVersion f@(Framewor = do (CachePrefix prefix, verbose) <- ask let finalRemoteBcsymbolmaploadPath = prefix remoteBcSymbolmapUploadPath - mapExceptT (withReaderT (const (verbose))) $ getArtifactFromEngine + mapExceptT (withReaderT (const verbose)) $ getArtifactFromEngine enginePath finalRemoteBcsymbolmaploadPath symbolmapName @@ -100,7 +100,7 @@ getDSYMFromEngine enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ = do (CachePrefix prefix, verbose) <- ask let finalRemoteDSYMUploadPath = prefix remoteDSYMUploadPath - mapExceptT (withReaderT (const (verbose))) + mapExceptT (withReaderT (const verbose)) $ getArtifactFromEngine enginePath finalRemoteDSYMUploadPath dSYMName where remoteDSYMUploadPath = remoteDsymPath platform reverseRomeMap f version @@ -215,7 +215,7 @@ getArtifactFromEngine :: FilePath -- ^ The `FilePath` to the engine -> FilePath -- ^ The remote path -> String -- ^ A colloquial name for the artifact - -> ExceptT String (ReaderT (Bool) IO) LBS.ByteString + -> ExceptT String (ReaderT Bool IO) LBS.ByteString getArtifactFromEngine enginePath remotePath artifactName = do readerEnv@(verbose) <- ask eitherArtifact :: Either IOError LBS.ByteString <- liftIO $ try $ runReaderT @@ -235,22 +235,22 @@ downloadBinaryWithEngine :: FilePath -- ^ The `FilePath` to the engine -> FilePath -> FilePath - -> (ReaderT (Bool) IO) LBS.ByteString + -> (ReaderT Bool IO) LBS.ByteString downloadBinaryWithEngine enginePath objectRemotePath objectName = do - (verbose) <- ask - let cmd = Turtle.fromString $ enginePath + verbose <- ask + let cmd = Turtle.fromString enginePath let sayFunc = if verbose then sayLnWithTime else sayLn when verbose $ sayLnWithTime $ "Invoking engine " - <> (show enginePath) + <> show enginePath <> " to download " <> objectName <> " from: " <> objectRemotePath - (exitCode) <- Turtle.proc + exitCode <- Turtle.proc cmd - ["download", (Turtle.fromString objectRemotePath), (Turtle.fromString objectRemotePath)] + ["download", Turtle.fromString objectRemotePath, Turtle.fromString objectRemotePath] (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () diff --git a/src/Engine/Probing.hs b/src/Engine/Probing.hs index f43c918..1fabbc4 100644 --- a/src/Engine/Probing.hs +++ b/src/Engine/Probing.hs @@ -56,15 +56,15 @@ probeEngineForFrameworkOnPlatform -> m PlatformAvailability probeEngineForFrameworkOnPlatform enginePath reverseRomeMap (FrameworkVersion fwn version) platform = do - let cmd = Turtle.fromString $ enginePath - (exitCode) <- Turtle.proc + let cmd = Turtle.fromString enginePath + exitCode <- Turtle.proc cmd - ["list", (Turtle.fromString remoteFrameworkUploadPath)] + ["list", Turtle.fromString remoteFrameworkUploadPath] (return $ Turtle.unsafeTextToLine "") case exitCode of -- If engine exits with success, we assume the framework exists. Turtle.ExitSuccess -> return (PlatformAvailability platform True) - Turtle.ExitFailure n -> return (PlatformAvailability platform False) + Turtle.ExitFailure _ -> return (PlatformAvailability platform False) where remoteFrameworkUploadPath = remoteFrameworkPath platform reverseRomeMap fwn version diff --git a/src/Engine/Uploading.hs b/src/Engine/Uploading.hs index 2599f4c..fe511a3 100644 --- a/src/Engine/Uploading.hs +++ b/src/Engine/Uploading.hs @@ -28,7 +28,7 @@ uploadFrameworkToEngine uploadFrameworkToEngine frameworkArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform = when (platform `elem` fwps) $ do (CachePrefix prefix, verbose) <- ask - withReaderT (const (verbose)) $ uploadBinary + withReaderT (const verbose) $ uploadBinary enginePath (Zip.fromArchive frameworkArchive) (prefix remoteFrameworkUploadPath) @@ -50,7 +50,7 @@ uploadDsymToEngine uploadDsymToEngine dSYMArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform = when (platform `elem` fwps) $ do (CachePrefix prefix, verbose) <- ask - withReaderT (const (verbose)) $ uploadBinary + withReaderT (const verbose) $ uploadBinary enginePath (Zip.fromArchive dSYMArchive) (prefix remoteDsymUploadPath) @@ -71,7 +71,7 @@ uploadBcsymbolmapToEngine uploadBcsymbolmapToEngine dwarfUUID dwarfArchive enginePath reverseRomeMap (FrameworkVersion f@(Framework fwn _ fwps) version) platform = when (platform `elem` fwps) $ do (CachePrefix prefix, verbose) <- ask - withReaderT (const (verbose)) $ uploadBinary + withReaderT (const verbose) $ uploadBinary enginePath (Zip.fromArchive dwarfArchive) (prefix remoteBcsymbolmapUploadPath) @@ -91,7 +91,7 @@ uploadVersionFileToEngine' uploadVersionFileToEngine' enginePath versionFileContent projectNameAndVersion = do (CachePrefix prefix, verbose) <- ask - withReaderT (const (verbose)) $ uploadBinary + withReaderT (const verbose) $ uploadBinary enginePath versionFileContent (prefix versionFileRemotePath) @@ -109,28 +109,27 @@ uploadBinary -> LBS.ByteString -> FilePath -> FilePath - -> ReaderT (Bool) a () + -> ReaderT Bool a () uploadBinary enginePath binaryZip destinationPath objectName = do - (verbose) <- ask - let cmd = Turtle.fromString $ enginePath + verbose <- ask + let cmd = Turtle.fromString enginePath liftIO $ saveBinaryToFile binaryZip destinationPath when verbose $ sayLnWithTime $ "Invoking engine " - <> (show enginePath) + <> show enginePath <> " to upload " <> destinationPath - (exitCode) <- Turtle.proc + exitCode <- Turtle.proc cmd - ["upload", (Turtle.fromString destinationPath), (Turtle.fromString destinationPath)] + ["upload", Turtle.fromString destinationPath, Turtle.fromString destinationPath] (return $ Turtle.unsafeTextToLine "") case exitCode of Turtle.ExitSuccess -> return () - Turtle.ExitFailure n -> do - sayLn + Turtle.ExitFailure n -> sayLn $ "Error " - <> (show n) + <> show n <> ": could not upload " <> destinationPath diff --git a/src/Lib.hs b/src/Lib.hs index b546d56..12f4f5a 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -215,8 +215,7 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do case command of Upload (RomeUDCPayload gitRepoNames platforms cachePrefixString skipLocalCache noIgnoreFlag noSkipCurrentFlag concurrentlyFlag) - -> do - sayVersionWarning romeVersion verbose + -> sayVersionWarning romeVersion verbose *> performWithDefaultFlow uploadArtifacts ( verbose @@ -326,12 +325,6 @@ runUDCCommand command absoluteRomefilePath verbose romeVersion = do <> romeVersionToString vers <> noColorControlSequence - -- case (mS3BucketName, mEnginePath) of - -- (Nothing, Nothing) -> undefined -- Proceed donw regular path - -- (Just _, Nothing) -> undefined -- Proceed donw regular path - -- (Just b , Just e) -> throwError $ "Cannot specify both bucket \"" ++ show b ++ "\" and engine at " ++ e ++ " at the same time." - -- (Nothing, Just e) -> undefined -- run command with all info - type FlowFunction = Maybe S3.BucketName -- ^ Just an S3 Bucket name or Nothing -> Maybe FilePath -- ^ Just the path to the local cache or Nothing -> Maybe FilePath -- ^ Just the path to the engine or Nothing From 18232e9446ac97cf2aae03c32bae0f79f310958b Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Fri, 17 May 2019 18:25:51 +0200 Subject: [PATCH 36/41] Fix copying of script when running on CI --- integration-tests/current-framework-engine-yml.bats | 2 +- integration-tests/current-framework-named-engine-yml.bats | 2 +- integration-tests/dynamic-frameworks-engine-yml.bats | 4 ++-- integration-tests/static-frameworks-engine-yml.bats | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/integration-tests/current-framework-engine-yml.bats b/integration-tests/current-framework-engine-yml.bats index 56cb114..3003808 100644 --- a/integration-tests/current-framework-engine-yml.bats +++ b/integration-tests/current-framework-engine-yml.bats @@ -8,7 +8,7 @@ setup() { mkdir -p $BATS_TMPDIR/Rome-Tests - cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cp integration-tests/engine.sh $BATS_TMPDIR/Rome-Tests/ cd $BATS_TMPDIR/Rome-Tests diff --git a/integration-tests/current-framework-named-engine-yml.bats b/integration-tests/current-framework-named-engine-yml.bats index 3658dbd..833fcc9 100644 --- a/integration-tests/current-framework-named-engine-yml.bats +++ b/integration-tests/current-framework-named-engine-yml.bats @@ -10,7 +10,7 @@ setup() { mkdir -p $BATS_TMPDIR/Rome-Tests - cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cp integration-tests/engine.sh $BATS_TMPDIR/Rome-Tests/ cd $BATS_TMPDIR/Rome-Tests diff --git a/integration-tests/dynamic-frameworks-engine-yml.bats b/integration-tests/dynamic-frameworks-engine-yml.bats index 43808b3..8370306 100644 --- a/integration-tests/dynamic-frameworks-engine-yml.bats +++ b/integration-tests/dynamic-frameworks-engine-yml.bats @@ -2,14 +2,14 @@ setup() { - export ALAMOFIRE_VERSION="4.7.3" + export ALAMOFIRE_VERSION="4.8.2" export RESULT_VERSION="4.0.0" rm -rf $BATS_TMPDIR/Rome-Tests mkdir -p $BATS_TMPDIR/Rome-Tests - cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cp integration-tests/engine.sh $BATS_TMPDIR/Rome-Tests/ cd $BATS_TMPDIR/Rome-Tests diff --git a/integration-tests/static-frameworks-engine-yml.bats b/integration-tests/static-frameworks-engine-yml.bats index fe6ac40..8a9fcae 100644 --- a/integration-tests/static-frameworks-engine-yml.bats +++ b/integration-tests/static-frameworks-engine-yml.bats @@ -9,7 +9,7 @@ setup() { mkdir -p $BATS_TMPDIR/Rome-Tests - cp engine.sh $BATS_TMPDIR/Rome-Tests/ + cp integration-tests/engine.sh $BATS_TMPDIR/Rome-Tests/ cd $BATS_TMPDIR/Rome-Tests From dd78bafc315c8af5395f1d4a112d01c5a08e9123 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Fri, 17 May 2019 20:32:55 +0200 Subject: [PATCH 37/41] =?UTF-8?q?Add=20space=20in=20error=20message=20?= =?UTF-8?q?=E2=80=9Ccachein=E2=80=9D=20->=20=E2=80=9Ccache=20in=E2=80=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- src/Lib.hs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Lib.hs b/src/Lib.hs index 12f4f5a..042a2c5 100644 --- a/src/Lib.hs +++ b/src/Lib.hs @@ -168,7 +168,7 @@ conflictingCachesMessage conflictingSkipLocalCacheOptionMessage :: String conflictingSkipLocalCacheOptionMessage - = "Error: only \"local\" defined as cache\ + = "Error: only \"local\" defined as cache \ \in your Romefile, but you have asked Rome to skip \ \this cache." From cd1eaa5e5c0a1b22a4580ef67ff2c0a3317d9b92 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Fri, 17 May 2019 20:33:13 +0200 Subject: [PATCH 38/41] Fix crash in reading INI configuration since engine is unsupported --- src/Data/Romefile.hs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Data/Romefile.hs b/src/Data/Romefile.hs index 5c447c8..b70d221 100644 --- a/src/Data/Romefile.hs +++ b/src/Data/Romefile.hs @@ -237,6 +237,7 @@ toRomefile ini = do let _engine = Nothing -- Engines are not supported in INI let _repositoryMapEntries = getRepositoryMapEntries ini _ignoreMapEntries = getIgnoreMapEntries ini + _enginePath = _engine _cacheInfo = RomeCacheInfo {..} Romefile <$> Right _cacheInfo From 9c453c987e88a321f05df8b27b12d6d4ec9b3fe6 Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 18 May 2019 08:29:11 +0200 Subject: [PATCH 39/41] Update README with mention and documentation for engine --- README.md | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 8d1f4c1..c1f8dae 100644 --- a/README.md +++ b/README.md @@ -9,6 +9,7 @@ Rome is a tool that allows developers on Apple platforms to use: - [Ceph](https://ceph.com/ceph-storage/object-storage/) - other S3 compatible object stores - or/and a local folder +- your own custom engine as a shared cache for frameworks built with [Carthage](https://github.com/Carthage/Carthage). @@ -34,6 +35,7 @@ Trusted by: - [Setting up AWS credentials](#setting-up-aws-credentials) - [Selecting the AWS Region](#selecting-the-aws-region) - [Setting up endpoint override for Minio, Ceph, or other S3 compatible stores](#setting-up-endpoint-override) + - [Custom Engine](#customengine) - [Romefile](#romefile) - [Cache](#cache) - [RepositoryMap](#repositorymap) @@ -257,6 +259,25 @@ Default port for `http` endpoints is __9000__ if the port is left unspecified. Alternatively the endpoint can also be specified by setting an `AWS_ENDPOINT` environment variable. +### Custom Engine +You can write your own script that Rome will use as engine to execute upload/download/list commands. You start by specifying the path to a script or executable in your [Romefile](#romefile) as shown in the example [structure](#structure). +Rome will invoke the specified script or executable with three commands and different parameters based on the action to perform: + +- `./script.sh upload local-path remote-path` +- `./script.sh download remote-path local-path` +- `./script.sh list remote-path` + +For example, if your [Romefile](#romefile) specifies `engine: script.sh`, Rome will execute the following command when uploading/downloading/listing a framework: +```sh +./script.sh upload Alamofire/iOS/Alamofire.framework-4.8.2.zip Alamofire/iOS/Alamofire.framework-4.8.2.zip +./script.sh download Alamofire/iOS/Alamofire.framework-4.8.2.zip Alamofire/iOS/Alamofire.framework-4.8.2.zip +./script.sh list Alamofire/iOS/Alamofire.framework-4.8.2.zip +``` + +The script should take the given `remote-path`, carry out its logic to retrieve the artifact and place it at `local-path`. Please refer to the [cache structure](#cachestructure) definition for more information on the cache is constructed. + +For an example of a custom engine, take a look at [engine.sh](https://github.com/blender/Rome/blob/master/integration-tests/engine.sh) which is used in the integration tests to simply copy artifacts in a different directory. Infinite uses cases are opened by using a custom engine, such as uploading artifacts to any non-compatible S3 storage system. + ### Romefile #### About the format @@ -299,10 +320,11 @@ A Romefile looks like this: ```yaml cache: # required - local: ~/Library/Caches/Rome # optional - # at least one between `local` and `s3Bucket` is required - s3Bucket: ios-dev-bucket # optional - # at least one between `local` and `s3Bucket` is required + # at least one of the following is required: + local: ~/Library/Caches/Rome # optional and can be combined with either a `s3Bucket` or `engine` + s3Bucket: ios-dev-bucket # optional and can be combined with `local` + engine: script.sh # optional and can be combined with `local` + repositoryMap: # optional - better-dog-names: # entry that does not follow # the "Organization/FrameworkName" convention. @@ -326,13 +348,16 @@ currentMap: The cache __must__ contain __at least one__ between: - the name of the S3 Bucket to upload/download to/from. The key `s3Bucket` is __optional__. - the path to local directory to use as an additional cache. The key `local` is __optional__. +- the path to a custom engine to use as an additional cache. The key `engine` is __optional__. ```yaml cache: # required local: ~/Library/Caches/Rome # optional - # at least one between `local` and `s3Bucket` is required + # at least one between `local`, `s3bucket` and `engine` is required s3Bucket: ios-dev-bucket # optional - # at least one between `local` and `s3Bucket` is required + # at least one between `local`, `s3bucket` and `engine` is required + engine: script.sh # optional + # at least one between `local`, `s3bucket` and `engine` is required ``` This is already a viable Romefile. @@ -491,7 +516,6 @@ The above means that `t1` is only available for `iOS` and `Mac`. The `--platforms` command line options can be used to futher limit the Rome command to a specific subset of the supported platfroms. - ### Cache Structure The following describes the structure of the cache that Rome creates and manages. From 62561c589f95a102c4bf64302e1629d9fc477ced Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 18 May 2019 21:01:35 +0200 Subject: [PATCH 40/41] Update README.md Co-Authored-By: Tommaso Piazza <196761+blender@users.noreply.github.com> --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index c1f8dae..78f8dfe 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Rome is a tool that allows developers on Apple platforms to use: - [Ceph](https://ceph.com/ceph-storage/object-storage/) - other S3 compatible object stores - or/and a local folder -- your own custom engine +- [your own custom engine](#customengine) as a shared cache for frameworks built with [Carthage](https://github.com/Carthage/Carthage). From 42ec412f229ba8edf21f3dbd10788880669fb59e Mon Sep 17 00:00:00 2001 From: Patrick Balestra Date: Sat, 18 May 2019 21:03:03 +0200 Subject: [PATCH 41/41] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 78f8dfe..a95c842 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ Rome is a tool that allows developers on Apple platforms to use: - [Ceph](https://ceph.com/ceph-storage/object-storage/) - other S3 compatible object stores - or/and a local folder -- [your own custom engine](#customengine) +- [your own custom engine](#custom-engine) as a shared cache for frameworks built with [Carthage](https://github.com/Carthage/Carthage).