You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sedona.apache.org by ji...@apache.org on 2022/07/13 00:50:00 UTC
[incubator-sedona] branch master updated: [DOCS] Create DownloadImageFromGEE.ipynb (#642)
This is an automated email from the ASF dual-hosted git repository.
jiayu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-sedona.git
The following commit(s) were added to refs/heads/master by this push:
new 610025db [DOCS] Create DownloadImageFromGEE.ipynb (#642)
610025db is described below
commit 610025db76775838939216a023fd1840903f8453
Author: Ana Caroline Ferreira <an...@gmail.com>
AuthorDate: Tue Jul 12 21:49:57 2022 -0300
[DOCS] Create DownloadImageFromGEE.ipynb (#642)
---
binder/DownloadImageFromGEE.ipynb | 282 ++++++++++++++++++++++++++++++++++++++
1 file changed, 282 insertions(+)
diff --git a/binder/DownloadImageFromGEE.ipynb b/binder/DownloadImageFromGEE.ipynb
new file mode 100644
index 00000000..1acd9c38
--- /dev/null
+++ b/binder/DownloadImageFromGEE.ipynb
@@ -0,0 +1,282 @@
+{
+ "cells": [
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "5ab40e60-e58c-4b50-a9ee-72bd875209ca",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# NO TERMINAL\n",
+ "# pip install earthengine-api\n",
+ "# pip install geemap\n",
+ "# pip install pywebhdfs"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "98df74a0-1730-4433-9ec8-8d3d77c54b50",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# DOWNLOAD FROM GEE"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "ed1bb739-a013-417d-9b2a-566ae89b0292",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import ee\n",
+ "import geemap\n",
+ "from pywebhdfs.webhdfs import PyWebHdfsClient\n",
+ "from datetime import date, timedelta\n",
+ "import os"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "id": "ffaa907b-3c69-461c-84a9-7e533ee8589c",
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "<p>To authorize access needed by Earth Engine, open the following\n",
+ " URL in a web browser and follow the instructions:</p>\n",
+ " <p><a href=https://accounts.google.com/o/oauth2/auth?client_id=517222506229-vsmmajv00ul0bs7p89v5m89qs8eb9359.apps.googleusercontent.com&scope=https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fearthengine+https%3A%2F%2Fwww.googleapis.com%2Fauth%2Fdevstorage.full_control&redirect_uri=urn%3Aietf%3Awg%3Aoauth%3A2.0%3Aoob&response_type=code&code_challenge=K-IM1uXz0IQDC8jBaT1Wg1f70o4hf6h3k9KgLmAZalI&code_challenge_method=S256>https://accounts.google.com/o/oauth2/auth?client_id=51722250 [...]
+ " <p>The authorization workflow will generate a code, which you\n",
+ " should paste in the box below</p>\n",
+ " "
+ ],
+ "text/plain": [
+ "<IPython.core.display.HTML object>"
+ ]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ },
+ {
+ "name": "stdin",
+ "output_type": "stream",
+ "text": [
+ "Enter verification code: 4/1AX4XfWgChJ0zb8Moe0TBT-joyNm6DNKHo6L18a0TkCyKTSFgnOFsGSwn92A\n"
+ ]
+ },
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Successfully saved authorization token.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Inicia o fluxo de autenticação\n",
+ "ee.Authenticate()\n",
+ "\n",
+ "# Inicializa a biblioteca\n",
+ "ee.Initialize()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "id": "2d866b25-8895-4d71-989a-8bde2c2becaa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# VARIAVEIS GLOBAIS (MUDAR AQUI!)\n",
+ "end_date = date.today()\n",
+ "percent_cloud = 100\n",
+ "# box = [[xmin, ymin], [xmin, ymax], [xmax, ymax], [xmax, ymin], [xmin, ymin]]\n",
+ "# https://boundingbox.klokantech.com/\n",
+ "box = [[[-54.6306579887,-25.5892766534],[-54.5393341362,-25.5892766534],[-54.5393341362,-25.4046299874],[-54.6306579887,-25.4046299874],[-54.6306579887,-25.5892766534]]]\n",
+ "boundary = ee.Geometry.Polygon(box, None, False)\n",
+ "collection_name = \"COPERNICUS/S2_SR\"\n",
+ "scale = 10\n",
+ "crs = 'EPSG:4326'\n",
+ "# CRIE UMA PASTA VAZIA NO DIRETORIO RASTER E COLOQUE O CAMINHO DELA AQUI EM BAIXO\n",
+ "out_dir = \"raster/sentinel2_tmp\"\n",
+ "hdfs_dir = \"sentinel2_tmp\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "68d64f0b-040a-4ccf-980e-0e8f201ce719",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Total number of images: 0\n",
+ "\n",
+ "Total number of images: 0\n",
+ "\n",
+ "Total number of images: 1\n",
+ "\n",
+ "Exporting 1/1: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/b15a6ed8d284266b94147d85f741ed13-1fcd749687233dd2cb56e7828e6d1bb8:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n",
+ "Total number of images: 1\n",
+ "\n",
+ "Exporting 1/1: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/b15a6ed8d284266b94147d85f741ed13-cb8dd97c1aeb7dc713e018e86fc54600:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n",
+ "Total number of images: 1\n",
+ "\n",
+ "Exporting 1/1: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/b15a6ed8d284266b94147d85f741ed13-f400c1a106bdb4b10e0a979634ebd16f:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n",
+ "Total number of images: 2\n",
+ "\n",
+ "Exporting 1/2: 20211221T134209_20211221T134205_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/3aa9a4dd0980782c6450b743424311da-d17433801f5712f60c47b17b71b4281c:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211221T134209_20211221T134205_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Exporting 2/2: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/b15a6ed8d284266b94147d85f741ed13-859f18c37de9efdab4faa18bd55e652d:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n",
+ "Total number of images: 2\n",
+ "\n",
+ "Exporting 1/2: 20211221T134209_20211221T134205_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/3aa9a4dd0980782c6450b743424311da-cef8e1649f70d212459136a58286fce1:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211221T134209_20211221T134205_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Exporting 2/2: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/be5e1a1d63296b074d488e4130f08578-58380cb69d77be8f9c45ae87ec164be5:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n",
+ "Total number of images: 3\n",
+ "\n",
+ "Exporting 1/3: 20211216T134211_20211216T134211_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/b63ccf2edf826be5501daa568c1fc35a-5fc6a742b7115b3978355a433962997c:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211216T134211_20211216T134211_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Exporting 2/3: 20211221T134209_20211221T134205_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/67c944bd3653197b6d1a48a8b7cd669f-01955ac97af9f3e738a80e65c7870921:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211221T134209_20211221T134205_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Exporting 3/3: 20211226T134211_20211226T134212_T21JYM.tif\n",
+ "Generating URL ...\n",
+ "Downloading data from https://earthengine.googleapis.com/v1alpha/projects/earthengine-legacy/thumbnails/7b96db8dad91ced9e4ca0e746ab92377-ddb37508d7bc43278abca3a0c610cc81:getPixels\n",
+ "Please wait ...\n",
+ "Data downloaded to /opt/workspace/raster/sentinel2_tmp/20211226T134211_20211226T134212_T21JYM.tif\n",
+ "\n",
+ "\n",
+ "Existem imagens na pasta\n"
+ ]
+ }
+ ],
+ "source": [
+ "for i in range(2, 15, 2):\n",
+ " start_date = str((end_date - timedelta(days=i)))\n",
+ " collection = ee.ImageCollection(collection_name) \\\n",
+ " .select(['B2', 'B3', 'B4', 'B8']) \\\n",
+ " .filterBounds(boundary) \\\n",
+ " .filterMetadata('CLOUDY_PIXEL_PERCENTAGE', 'less_than', percent_cloud) \\\n",
+ " .filterDate(start_date, str(end_date))\n",
+ " geemap.ee_export_image_collection(collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir)\n",
+ "# NOT SURE WHAT F.. THIS BELOW DOES \n",
+ " for root, directory, files in os.walk(out_dir):\n",
+ " if len(files) == 0:\n",
+ " geemap.ee_export_image_collection(collection, scale=scale, crs=crs, region=boundary, out_dir=out_dir)\n",
+ " else:\n",
+ " print('Existem imagens na pasta')\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "25c9d1be-9a86-4ec5-bd3d-4f23ee2ca024",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "hdfs = PyWebHdfsClient(host='179.106.229.159',port='50070', user_name='root')\n",
+ "hdfs.delete_file_dir(hdfs_dir, recursive=True)\n",
+ "hdfs.make_dir(hdfs_dir)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "e0961125-15bf-4ff2-b851-e7b90a177c16",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Adicionar imagens no HDFS\n",
+ "\n",
+ "# No terminal do JupyterLab\n",
+ "# sudo docker exec -itu 0 hadoop passwd (primeira vez) - hoje é admin123\n",
+ "# ssh hadoop\n",
+ "# cd ..\n",
+ "# cd hadoop/bin\n",
+ "# ./hadoop fs -copyFromLocal /opt/workspace/raster/sentinel2_tmp/* /sentinel2_tmp"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.9.2"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}