NOTES.sh

# Copied from env22:

''' Environment variables for image processing and classification for 2022 work.

Order of operations, prior to automating in python!:
0. Be sure to set environment vars: batch (number or None); mosaic
1. Download data
    $ landsatxplore  download -o . LC08_L2SP_041036_20140516_20200911_02_T1
2. Create 3-band images using gdal (and make COG)
    $ BASE=LT05_L1TP_070014_19850831_20200918_02_T1
    $ ROI=redberry
    $ gdal_merge.py -co TILED=YES -co COMPRESS=LZW -seperate /vsitar/download/"$BASE".tar/"$BASE"_SR_B5.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B3.TIF /vsitar/download/"$BASE".tar/"$BASE"_SR_B4.TIF -o redberry/merge-sr/"$BASE"_SR_B534.tif \
    $ && \
    $ gdal_translate -of COG redberry/merge-sr/"$BASE"_SR_B534.tif  redberry/merge-sr/COG/"$BASE"_SR_B534.tif
3. Run rescale-landsat.py to create new merged-sr-uint8 directory
4. Use command line variables and gdal to crop to /mnt/gcs/cropped-to-roi/uint8
    $ f=/mnt/gcs/landsat8/rhode-island/merge-sr/LC08_L2SP_012031_20201129_20210316_02_T1_SR_B534.tif
    $ shp=/mnt/gcs/roi/lincoln-roi.shp
    $ region=_L
    $ gdalwarp -cutline $shp -crop_to_cutline -dstnodata 0 -co TILED=YES $f /mnt/gcs/cropped-to-roi/`basename $f .$ tif`"$region".tif
5. Run pixel-smasher/old_BasicSR/codes/utils/chunk_images.py
6. On second VM: Run SR, but be sure to set proper input (path with batch number) in its options yml file.
7. On second VM: Move SR
    $ b=1 # batch number
    $ ls results/Landsat_SRx10/visualization/batch$b/*.png > results/Landsat_SRx10/visualization/batch$b/file_names.txt
    $ mv results/Landsat_SRx10/visualization/batch$b/*.png /mnt/gcs/sr/v2/10x/batch$b
8. Re-chunk: set sr_dir

Additional notes on local in scratch.txt and remote in NOTES.sh.'''

20200423_082122_06_1065_3B_AnalyticMS_SR_s0118_45000.png

for file in .; do echo ${file%.png}

find /data_dir/valid_mod/ -name "*20200423_082122_06_1065_3B_AnalyticMS_SR_s0118*"

cp `find visualization/ -name "*45000.png" | shuf -n 2000` ~/data_dir/visualization &

for file in `ls`; do cp /data_dir/valid_mod/HR/x4/${file%_400000.png}.png ../valid_mod/HR/x4/; done 

find visualization/20190805_054900_71_105d_3B_AnalyticMS_SR_s0469 -name "20190805_054900_71_105d_3B_AnalyticMS_SR_s0469_[1,2,3,5,6,7,8,9,0]*.png"

# sudo chmod a+rwx $file; sudo chown ethan_kyzivat: $file; 

# for deleting file at weird checkpoint times- keeps all files with iteration that begins with 4 (corr. to final 400,000 and 400,004 iterations and "worst" 40,000 - for run 0043), and also the 5,000 iter first image and 200,000 midpoint

cd /mnt/disks/extraspace/pixel-smasher/experiments/006_ESRGAN_x4_PLANET_noPreTrain_wandb_sep28
for file in `find visualization/*/ -name "*s*_[1,3,6,7,8,9,0]*.png"`; do sudo rm $file; done & # removds most * (except 2,3,4, 5)
for file in `find visualization/*/ -name "*s*_[2][1-9][1-9]*.png"`; do sudo rm $file; done & # removes most 2*
# for file in `find visualization/*/ -name "*s*_3[0,2-9]*.png"`; do sudo rm $file; done & # removes most 3*, but keeps latest 310,000 iter # toggle this
for file in `find visualization/*/ -name "*s*_2????.png"`; do sudo rm $file; done & # removes 20,000, but keeps 200,000 midpoint
for file in `find visualization/*/ -name "*s*_5????.png"`; do sudo rm $file; done & # removes 50,000

## to rm additional in the 200,000's:
for file in `find visualization/*/ -name "*s*_2[1-9]????.png"`; do sudo rm $file; done & 

## to rm files from 008_ESRGAN_x10_PLANET_noPreTrain_wandb_2GPUs_Oct8_VGG160: (I want to save 2500, 70000, 137500)
dl /data_dir/pixel-smasher/experiments/keep/008_ESRGAN_x10_PLANET_noPreTrain_wandb_2GPUs_Oct8_VGG160

# for file in `find visualization/*/ -name "*s*_[3,4,5,6,8,9,0]*.png"`; do sudo rm $file; done & # removds most * # error arg list too long...
for file in `find visualization/ -name "*s*_[3]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[4]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[5]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[6]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[8]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[9]*.png"`; do sudo rm $file; done &
for file in `find visualization/ -name "*s*_[0]*.png"`; do sudo rm $file; done &

for file in `find visualization/ -name "*s*_1[1,2,4,5,6,7,8,9,0]*.png"`; do sudo rm $file; done & # rm most [1]*, but saves 13*
for file in `find visualization/ -name "*s*_13[1,2,3,4,5,6,8,9,0]*.png"`; do sudo rm $file; done & # rm most [1]*, but saves 137*

for file in `find visualization/ -name "*s*_2[1,2,3,4,6,7,8,9,0]*.png"`; do sudo rm $file; done & # rm most [2]*, but saves 25*
for file in `find visualization/ -name "*s*_25000.png"`; do sudo rm $file; done & # rm 25000

for file in `find visualization/ -name "*s*_7[1,2,3,4,5,6,7,8,9]*.png"`; do sudo rm $file; done & # rm most [7]*, but saves 70000

## workflow:
'''python /home/ethan_kyzivat/code/pixel-smasher/old_BasicSR/codes/scripts/extract_subimgs_single.py && bash /home/ethan_kyzivat/code/pixel-smasher/old_BasicSR/codes/utils/rand_shuf.sh && python /home/ethan_kyzivat/code/pixel-smasher/old_BasicSR/codes/scripts/generate_mod_LR_bic_parallel.py

We noticed youre using a conda environment. If you are experiencing issues with this environment in the integrated terminal, we recommend that you let the Python extension change "terminal.integrated.inheritEnv" to false in your user settings.
'''
# To update colorinterp of tif subsets snipped from ArcGis

for file in /data_dir/Scenes-shield-gt-subsets/*; do 
    gdal_translate  -colorinterp blue,green,red,alpha $file /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets/`basename $file '.tif'`.tif
done

# to create masks: use gdal_reproject_match to make pekel mask subsets, then put through normal generate_mod_LR_res.py process.

export PATH=$PATH:/home/ethan_kyzivat/code/geographic-functions
masks_dir='/data_dir/Shield_Water_Mask/Scenes-shield-gt'
out_dir='/data_dir/Shield_Water_Mask/Scenes-shield-gt-subsets'
for file in /data_dir/Scenes-shield-gt-subsets/*; do 
    IN=$masks_dir/${file:35:37}_no_buffer_mask.tif # e.g. 20170709_180516_1005_3B_AnalyticMS_SR + '_no_buffer_mask.tif'
    OUT=$out_dir/`basename $file .tif`.tif
    echo Input: $IN
    ########### uncomment for testing
    # ls $IN
    # echo OUT: $OUT
    # ls $file
    ###################

    bash gdal_reproject_match.sh $IN $OUT $file
done

# to convert from .tif to .png:
for file in /data_dir/Shield_Water_Mask/Scenes-shield-gt-subsets/*; do 
    gdal_translate -a_nodata none $file /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_masks/`basename $file '.tif'`.png
done

# to convert to [0, 255]:
for file in /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_masks_01/*.png; do 
    # gdal_calc.py -A $file --calc="A*255" --outfile=/data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_masks/`basename $file`
    ls $file
done

# to convert from .tif to .png (after katia work):
for file in /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_masks_0_255_tif/*; do 
    gdal_translate -a_nodata none $file /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_masks/`basename $file '.tif'`.png
done

# to convert from 16 bit to 8 bit: # HERE do

for file in /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets/*.png; do  
    gdal_translate -a_nodata none -ot Byte -scale 0 65535 $file /data_dir/planet_sub/hold_mod_scenes-shield-gt-subsets_uint8/`basename $file`
done

dl /data_dir/pixel-smasher/experiments/008_ESRGAN_x10_PLANET_noPreTrain_wandb_2GPUs_Oct8_VGG160
for dir in `find visualization -type d | shuf -n 100`; do
    echo $dir
    cp -r $dir visualization-complete
    echo copied!
done

# TensorBoard >= 2.4.1 and the PyTorch Profiler TensorBoard Plugin are required. Would you like to install these packages?
# Command pallette: tensorboard

## mosaic using gdal_merge
mkdir -p reproj mosaics
for file in LC08_L2SP_068014_20170708_20200903_02_T1_SR_B534_YF.tif LC08_L2SP_068013_20170708_20201015_02_T1_SR_B534_YF.tif ; do 
    gdalwarp $file reproj/$file -r cubic -t_srs reproj/102001.prj  -overwrite -wo NUM_THREADS=2 -multi
done
gdal_merge.py -o mosaics/LC08_2017_yflats.tif -co COMPRESS=LZW -n 0 -a_nodata 0 reproj/LC08_L2SP_068014_20170708_20200903_02_T1_SR_B534_YF.tif reproj/LC08_L2SP_068013_20170708_20201015_02_T1_SR_B534_YF.tif # ls LC08_*20170708*
rm reproj/*.tif

## Or

for file in LC08_L2SP_064012_20170728_20200903_02_T1_SR_B534_IK.tif LC08_L2SP_064011_20170728_20200903_02_T1_SR_B534_IK.tif ; do 
    gdalwarp $file reproj/$file -r cubic -t_srs reproj/102001.prj  -overwrite -wo NUM_THREADS=2 -multi
done
gdal_merge.py -o mosaics/LC08_2017_ocflats.tif -co COMPRESS=LZW -n 0 -a_nodata 0 LC08_L2SP_064012_20170728_20200903_02_T1_SR_B534_IK.tif LC08_L2SP_064011_20170728_20200903_02_T1_SR_B534_IK.tif

## renaming files
for f in old-crow-flats-roi.*; do mv $f ` echo $f | awk '{ gsub(/old-crow-flats/,"ocflats"); print '}`; done

## Sync ROIS:
# cd /mnt/e/My Drive/Research/GAN project/remote-data/individ_roi/renamed
gsutil -m  rsync -x .*desktop.ini . gs://ncrl-gan-working/roi/
## move files after SR run to GCS
# mv results/Landsat_SRx10/visualization/batch1/*.png /mnt/gcs/sr/v2/10x/batch1
mv results/Landsat_SRx10/visualization/batch1/*.png /mnt/gcs/sr/v2/batch1

## Zip each shapefile and assoc files in dir
for f in *.shp; do find . -name "`basename $f .shp`*" -print | zip -@ `basename $f .shp`.zip; done

## Move files that shouldn't have been chunked out of batch2 (the slow way)
mv chips/batch2/LC08_L2SP_064012_20170728_20200903_02_T1*.tif trash/batch2-shd-have-been-mosaicked/ & mv chips/batch2/LC08_L2SP_064011_20170728_20200903_02_T1*.tif trash/batch2-shd-have-been-mosaicked/ &

## Move the fast way
# gsutil -m mv 'gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_06801[34]_20170708*.tif' gs://ncrl-gan-working/cropped-to-roi/uint8/trash/batch2-shd-have-been-mosaicked/
gsutil -m mv 'gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_050015_20170811_20200903_02_T1*.tif' gs://ncrl-gan-working/cropped-to-roi/uint8/trash/batch2-shd-have-been-mosaicked/ &
gsutil -m mv 'gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_048016_20170829_20200903_02_T1*.tif' gs://ncrl-gan-working/cropped-to-roi/uint8/trash/batch2-shd-have-been-mosaicked/ &
gsutil -m mv 'gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_041021_20170812_20200903_02_T1*.tif' gs://ncrl-gan-working/cropped-to-roi/uint8/trash/batch2-shd-have-been-mosaicked/ & # potholes
gsutil -m mv 'gs://ncrl-gan-working/cropped-to-roi/uint8/chips/batch2/LC08_L2SP_04102[12]_20170812_20200903_02_T1*.tif' gs://ncrl-gan-working/cropped-to-roi/uint8/trash/batch2-shd-have-been-mosaicked/ & # potholes

## Copy files from KAtia VM to bucket after running SR (on katia VM)

cd /home/ekaterina_lezine_alumni_brown_ed/pixel-smasher/results
/home/ekaterina_lezine_alumni_brown_ed/pixel-smasher/results/Landsat_SRx10/visualization/batch2/LC08_L2SP_045015_20170723_20201015_02_T1_SR_B534_DA_y3232x2528_3m.png

# ## Output combined SR is in (SR run directory path)
# /mnt/gcs/sr/v1/overlap_16/combined_scale_30
# /mnt/gcs/sr/v2/10x/overlap_16/batch1/combined_scale_30 # Should be the following, but I deleted: '/home/ekaterina_lezine_alumni_brown_ed/pixel-smasher/results/Landsat_SRx10_archived_20220430_204102/visualization/batch1'
# /mnt/gcs/sr/v2/10x/overlap_16/batch2/combined_scale_30/ # Not yet

# ## SR results
# Landsat_Test_archived_20220428_211109 # has 3185 files Katia ran (v1)
# Landsat_SRx10_archived_20220430_204102 # Batch 1: my first successfull run: includes mosaics
# Landsat_SRx10/ # Batch 2: to be renamed # my second successfull fun

# ## Re-chunking run results
# v1          x
# v2, batch 1 o   (only some mosaics made so far: Inuvik)
# v2, batch 2 o   (no mosaics made so far, but lots of SR)

## Make COG with overviews

for f in *.tif; do echo Making COG: $f; gdal_translate -of COG $f cog/$f; done
# gdaladdo -ro cog/$f; # not necessary

## calculate data extent bbox/raster domain and crop to remove fill columns: gdal translate wants order: [-projwin ulx uly lrx lry], so order in rio shapes output: 1-4-3-2
file=mosaics/LC08_20170708_yflats.tif
# rio shapes --bbox --projected --mask $file
bbox=(`rio shapes --bbox --projected --mask $file | sed 's/\[//g' | sed 's/\]//g' | sed 's/,//g'`)
# gdal_translate -a_nodata 0 -co TILED=YES -co COMPRESS=LZW -projwin 391395.0 7207155.0 481335.0 7070625.0 $file ${file%.*}_cropped.tif
gdal_translate -a_nodata 0 -co TILED=YES -co COMPRESS=LZW -projwin ${bbox[0]} ${bbox[3]} ${bbox[2]} ${bbox[1]} $file ${file%.*}_cropped.tif