Airflow - Study Notes 4

 

 

To retrieve these images, I frequently make use of the NASA Astronomy Picture of the Day API (https://apod.nasa.gov/apod/astropix.html) to gather a new image. This is a free API requiring an API key to be created but is easily accessible.

 

 

 

 

import requests
import json
from datetime import date

 

from NASA_Keys import api_key

url = f'https://api.nasa.gov/planetary/apod?api_key={api_key}'
response = requests.get(url).json()
response

 

{'date': '2025-01-19',
 'explanation': "What would it look like to land on Saturn's moon Titan? The European Space Agency's Huygens probe set down on the Solar System's cloudiest moon in 2005, and a time-lapse video of its descent images was created. Huygens separated from the robotic Cassini spacecraft soon after it achieved orbit around Saturn in late 2004 and began approaching Titan. For two hours after arriving, Huygens plummeted toward Titan's surface, recording at first only the shrouded moon's opaque atmosphere. The computerized truck-tire sized probe soon deployed a parachute to slow its descent, pierced the thick clouds, and began transmitting images of a strange surface far below never before seen in visible light. Landing in a dried sea and surviving for 90 minutes, Huygen's returned unique images of a strange plain of dark sandy soil strewn with smooth, bright, fist-sized rocks of ice.",
 'media_type': 'video',
 'service_version': 'v1',
 'title': 'Titan Touchdown: Huygens Descent Movie',
 'url': 'https://www.youtube.com/embed/msiLWxDayuA?rel=0'}

 

today_image = 'https://apod.nasa.gov/apod/image/2311/Kirkjufell2023Nov9_1024.jpg'
r = requests.get(today_image)
with open(f'todays_image_{date.today()}.png', 'wb') as f:
    f.write(requests.get(today_image).content)

 

import json
import pathlib
import airflow
import requests
import requests.exceptions as request_exceptions
from datetime import date
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.decorators import task
from datetime import datetime, timedelta


dag_owner = 'Frank'

 

def _get_pictures():
    pathlib.Path("/tmp/images").mkdir(parents=True,
    exist_ok=True)
    api_key = 'xxxxxxxxxxxxxxxxxxxxxxxxxxx'
    url = f'https://api.nasa.gov/planetary/apod?api_key={api_key}'
    response = requests.get(url).json()
    # today_image = response['url']
    today_image = 'https://apod.nasa.gov/apod/image/2311/Kirkjufell2023Nov9_1024.jpg'
    with open(f'todays_image_{date.today()}.png', 'wb') as f:
        f.write(requests.get(today_image).content)

 

default_args = {'owner': dag_owner,
    'depends_on_past': False,
    'retries': 2,
    'retry_delay': timedelta(minutes=5)
}

 

with DAG(dag_id='download_APOD_image',
    default_args=default_args,
    description='download and notify ',
    start_date = airflow.utils.dates.days_ago(0),
    schedule_interval='@daily',
    catchup=False,
    tags=['None']
):

 

    get_pictures = PythonOperator(
        task_id="get_pictures",
        python_callable=_get_pictures,
    )

 

    notify = BashOperator(
        task_id="notify",
        bash_command='echo f"Image for today has been added!"',
    )

 

    get_pictures >> notify

 

 

import json
import pathlib
import airflow
import requests
import requests.exceptions as request_exceptions
from datetime import date
from airflow import DAG
from airflow.operators.bash import BashOperator
from airflow.operators.python import PythonOperator
from airflow.decorators import task
from datetime import datetime, timedelta


dag_owner = "Frank"

default_args = {
    "owner": dag_owner,
    "depends_on_past": False,
    "retries": 2,
    "retry_delay": timedelta(minutes=5),
}


def _get_pictures():
    pathlib.Path("/tmp/images").mkdir(parents=True, exist_ok=True)
    api_key = "w9meQXZnGW7SJgN8vVUdV5uKriWTXobbMx6YGTm4"
    url = f"https://api.nasa.gov/planetary/apod?api_key={api_key}"
    response = requests.get(url).json()
    # today_image = response['url']
    today_image = "https://apod.nasa.gov/apod/image/2311/Kirkjufell2023Nov9_1024.jpg"
    with open(f"todays_image_{date.today()}.png", "wb") as f:
        f.write(requests.get(today_image).content)


with DAG(
    dag_id="download_APOD_image",
    default_args=default_args,
    description="download and notify ",
    start_date=airflow.utils.dates.days_ago(0),
    schedule_interval="@daily",
    catchup=False,
    tags=["None"],
):
    get_pictures = PythonOperator(
        task_id="get_pictures",
        python_callable=_get_pictures,
    )

    notify = BashOperator(
        task_id="notify",
        bash_command='echo f"Image for today has been added!"',
    )

    get_pictures >> notify