Setup the working space

 

1. Create anaconda environment

conda create --name scrappers --clone base

2. Activate environment

conda activate scrappers

3. Install key libs

$ conda config --add channels conda-forge
$ conda install cookiecutter

sudo apt-get install tree

Docker [Unverified] https://docs.docker.com/engine/install/ubuntu/
curl -fsSL https://get.docker.com -o get-docker.sh
DRY_RUN=1 sh ./get-docker.sh

4. Create cookiecutter project

mkdir defproj

cd defproj

cookiecutter -c v1 https://github.com/drivendata/cookiecutter-data-science

mkdir project_name

cd project_name

mkdir docker

5. Download Dockerfiles

curl -o Dockerfile https://raw.githubusercontent.com/mohamedabolfadl/default_docker_datascience/main/Dockerfile

cp Dockerfile project_name/docker/Dockerfile

curl -o project_name/docker/main.py https://raw.githubusercontent.com/mohamedabolfadl/default_docker_datascience/main/main.py

6. Initialize git repository [Not working at the push stage]

git init

git add .
git commit -m "first commit"
git branch -M main
git remote add origin https://github.com/USERNAME/REPONAME.git

git remote -v
git remote set-url origin git@github.com:USERNAME/REPONAME.git
git push -u origin main

6. Run Docker
docker build -t scrapper .

docker run -it -p 8080:8080 --mount type=bind,source=/home/mohamed/Desktop/old/defproj,target=/app  scrapper