IO & Data Access

This module takes care of receiving the data catalogs.

source

postprocessor

def postprocessor(
    fname, # Full path of the zipped file in local storage
    action, # One of "download" (file doesn't exist and will download),
    # "update" (file is outdated and will download), and
    # "fetch" (file exists and is updated so no download).
    pup, # The instance of Pooch that called the processor function.
): # The full path to the unzipped file. (Return the same fname is your
processor doesn't modify the file).

Post-processing hook to process the downloaded v1 catalog file.

Postprocessing steps are: 1. unzip to CSV 2. load CSV into pandas 3. save as parquet file.

This time, at the next read, the performance is much increased due to reading the parquet file instead of loading the CSV file every time.

v1.fetch("fans", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v1/fans.unzip/P4_catalog_v1.1_L1C_cut_0.5_fan.parq')
v1.fetch("blotches", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v1/blotches.unzip/P4_catalog_v1.1_L1C_cut_0.5_blotch.parq')
pd.read_parquet(v1.fetch("metadata", **kwargs))
OBSERVATION_ID IMAGE_CENTER_LATITUDE IMAGE_CENTER_LONGITUDE SOLAR_LONGITUDE START_TIME map_scale north_azimuth # of tiles
0 ESP_011296_0975 -82.1965 225.2530 178.833 2008-12-23 16:15:26 1.00 110.600107 91
1 ESP_011341_0980 -81.7969 76.1304 180.809 2008-12-27 04:25:02 0.50 110.208923 126
2 ESP_011348_0950 -85.0427 259.0940 181.117 2008-12-27 17:29:17 1.00 123.624057 91
3 ESP_011350_0945 -85.2160 181.4150 181.205 2008-12-27 21:14:01 0.50 99.672793 126
4 ESP_011351_0945 -85.2157 181.5480 181.249 2008-12-27 23:05:54 1.00 127.960688 91
... ... ... ... ... ... ... ... ...
216 ESP_022273_0950 -84.8184 65.7963 282.247 2011-04-28 00:21:01 0.25 117.236648 270
217 ESP_022339_0935 -86.2751 99.6123 285.423 2011-05-03 03:45:55 0.25 153.768164 315
218 ESP_022379_0930 -87.0045 86.5510 287.340 2011-05-06 06:34:07 0.25 153.003708 260
219 ESP_022510_0980 -81.6988 66.3188 293.572 2011-05-16 11:34:43 0.50 107.877998 72
220 ESP_022699_0985 -81.4043 295.7820 302.421 2011-05-31 05:01:48 0.50 96.757068 90

221 rows × 8 columns

v3.fetch("fans", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v3/fans.parq')
v3.fetch("blotches", **kwargs)
PosixPath('/Users/maye/Library/Caches/p4tools/v3/blotches.parq')
v3.fetch("tile_coords")
'/Users/maye/Library/Caches/p4tools/v3/tile_coords'

source

catalog_version

def catalog_version(
    version:str, # Catalog version string, e.g. ``'v3.1'``, ``'v3'``, ``'v1'``.
):

Context manager to temporarily use a different catalog version.

source

set_catalog_version

def set_catalog_version(
    version:str, # Catalog version string, e.g. ``'v3.1'``, ``'v3'``, ``'v1'``.
):

Set the default catalog version for the current session/notebook.

source

define_martian_year

def define_martian_year(
    df, # DataFrame to modify
    time_col_name, # Name of the time column to use for MY calculation
): # DataFrame with added 'MY' column

Add Martian Year (MY) column to dataframe based on time column.

source

get_tile_urls

def get_tile_urls()->DataFrame:

only v1 exists

source

get_region_names

def get_region_names(
    version:NoneType=None
)->DataFrame:

Per-obsid ROI assignments.

v3.1 (default) includes the 11 MY-33 obsids that are missing from the v1 mapping (5 Manhattan_Classic + 6 Ithaca). v1 is kept available for historical reproduction.

source

get_meta_data

def get_meta_data(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_tile_coords

def get_tile_coords(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_fan_catalog

def get_fan_catalog(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_blotch_catalog

def get_blotch_catalog(
    version:NoneType=None
)->DataFrame:

Call self as a function.

source

get_metafull

def get_metafull(
    version:NoneType=None
)->DataFrame:

only v3 and v3.1 exist

fans = get_fan_catalog()
fans["MY"].value_counts().sort_index()
MY
28     84431
29    100795
30     50941
31     15525
32      7470
33      6227
Name: count, dtype: int64
meta = get_metafull()
[col for col in meta.columns if "TIME" in col]
['OBSERVATION_START_TIME',
 'START_TIME',
 'STOP_TIME',
 'DELTA_LINE_TIME_COUNT',
 'LOCAL_TIME']
meta.columns
Index(['OBSERVATION_ID', 'VOLUME_ID', 'FILE_NAME_SPECIFICATION',
       'INSTRUMENT_HOST_ID', 'INSTRUMENT_ID', 'PRODUCT_ID',
       'PRODUCT_VERSION_ID', 'HICAL_VERSION', 'TARGET_NAME', 'ORBIT_NUMBER',
       'MISSION_PHASE_NAME', 'RATIONALE_DESC', 'OBSERVATION_START_TIME',
       'OBSERVATION_START_COUNT', 'START_TIME', 'SPACECRAFT_CLOCK_START_COUNT',
       'STOP_TIME', 'SPACECRAFT_CLOCK_STOP_COUNT', 'CCD_NAME',
       'CHANNEL_NUMBER', 'FILTER_NAME', 'SCAN_EXPOSURE_DURATION',
       'DELTA_LINE_TIME_COUNT', 'BINNING', 'TDI', 'TRIM_LINES',
       'FOCUS_POSITION_COUNT', 'FELICS_COMPRESSION_FLAG',
       'STIMULATION_LAMP_FLAG_RED', 'STIMULATION_LAMP_FLAG_BLUEGREEN',
       'STIMULATION_LAMP_FLAG_NEARINFRARED', 'LOOKUP_TABLE_TYPE',
       'LOOKUP_TABLE_MINIMUM', 'LOOKUP_TABLE_MAXIMUM', 'LOOKUP_TABLE_MEDIAN',
       'LOOKUP_TABLE_K_VALUE', 'LOOKUP_TABLE_NUMBER',
       'ADC_CONVERSION_SETTINGS_1', 'ADC_CONVERSION_SETTINGS_2',
       'FPA_POSITIVE_Y_TEMPERATURE', 'FPA_NEGATIVE_Y_TEMPERATURE',
       'FPE_TEMPERATURE', 'IEA_TEMPERATURE', 'IEA_PWS_BOARD_TEMPERATURE',
       'CPMM_PWS_BOARD_TEMPERATURE', 'IMAGE_LINES', 'LINE_SAMPLES',
       'SAMPLE_BITS', 'SCALED_PIXEL_WIDTH', 'EMISSION_ANGLE',
       'INCIDENCE_ANGLE', 'PHASE_ANGLE', 'IMAGE_CENTER_LATITUDE',
       'IMAGE_CENTER_LONGITUDE', 'MINIMUM_LATITUDE', 'MAXIMUM_LATITUDE',
       'MINIMUM_LONGITUDE', 'MAXIMUM_LONGITUDE', 'SPACECRAFT_ALTITUDE',
       'TARGET_CENTER_DISTANCE', 'SLANT_DISTANCE', 'NORTH_AZIMUTH',
       'SUB_SOLAR_AZIMUTH', 'SUB_SOLAR_LATITUDE', 'SUB_SOLAR_LONGITUDE',
       'SUB_SPACECRAFT_LATITUDE', 'SUB_SPACECRAFT_LONGITUDE', 'SOLAR_DISTANCE',
       'SOLAR_LONGITUDE', 'LOCAL_TIME', 'STEREO_FLAG', 'north_azimuth',
       '# of tiles', 'map_scale', 'MY'],
      dtype='str')
get_blotch_catalog("v1").tail()
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
249796 ESP_022699_0985 B044be7 0.00 APF0000idg 1276.20 14257.20 5 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.906430 -455.912914 -3342.551722 -81.388876 -81.488458 295.750015 30
249797 ESP_022699_0985 B044be8 0.00 APF0000idg 1311.25 14345.75 4 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.897692 -455.963924 -3342.536671 -81.388131 -81.487722 295.746616 30
249798 ESP_022699_0985 B044be9 0.00 APF0000idg 1266.00 14355.25 4 10.00 10.0 1.0 ... 302.421 0.5 96.757068 219.925196 -455.960274 -3342.537242 -81.387987 -81.487580 295.749599 30
249799 ESP_022699_0985 B044bea 168.67 APF0000idh 74.07 148.86 7 16.84 14.1 1.0 ... 302.421 0.5 96.757068 218.194755 -448.614747 -3343.589608 -81.514080 -81.612257 295.937105 30
249800 ESP_022699_0985 B044beb 172.85 APF0000idh 265.00 98.33 3 40.55 25.6 1.0 ... 302.421 0.5 96.757068 218.086509 -448.624657 -3343.607336 -81.514768 -81.612938 295.925426 30

5 rows × 24 columns

get_region_names()
obsid lat_IND lon_IND roi_name minimal_distance lat_WORD lon_WORD time MY
0 ESP_012079_0945 -85.4074 103.9700 Macclesfield 0.944588 -85.401 103.901 2009-02-22 16:32:19 29
1 ESP_021494_0945 -85.4041 103.9430 Macclesfield 0.511133 -85.401 103.901 2011-02-26 07:30:27 30
2 ESP_011407_0945 -85.4065 103.9830 Macclesfield 0.957066 -85.401 103.901 2009-01-01 07:49:15 29
3 ESP_020782_0945 -85.4071 103.9670 Macclesfield 0.901696 -85.401 103.901 2011-01-01 19:57:51 30
4 ESP_020716_0945 -85.4086 103.9790 Macclesfield 1.098928 -85.401 103.901 2010-12-27 16:32:24 30
... ... ... ... ... ... ... ... ... ...
462 ESP_047653_0950 -85.0542 180.2900 Ithaca 0.416589 -85.128 180.700 2016-09-25 14:35:20 33
463 ESP_047696_0935 -86.3881 99.0205 Manhattan_Classic 0.020588 -86.390 99.000 2016-09-28 23:00:27 33
464 ESP_048431_0950 -85.0536 180.2880 Ithaca 0.418664 -85.128 180.700 2016-11-25 05:46:04 33
465 ESP_048576_0950 -85.0559 180.3070 Ithaca 0.399559 -85.128 180.700 2016-12-06 12:58:38 33
466 ESP_049209_0950 -85.1548 180.7940 Ithaca 0.097746 -85.128 180.700 2017-01-24 20:49:23 33

467 rows × 9 columns

get_tile_urls()
tile_id tile_url
0 APF0000coq http://www.planetfour.org/subjects/standard/50...
1 APF0000cro http://www.planetfour.org/subjects/standard/50...
2 APF0000cs6 http://www.planetfour.org/subjects/standard/50...
3 APF0000co1 http://www.planetfour.org/subjects/standard/50...
4 APF0000cpd http://www.planetfour.org/subjects/standard/50...
... ... ...
114866 APF0002eep https://www.planetfour.org/subjects/standard/5...
114867 APF0002ee9 https://www.planetfour.org/subjects/standard/5...
114868 APF0002eej https://www.planetfour.org/subjects/standard/5...
114869 APF0002ee0 https://www.planetfour.org/subjects/standard/5...
114870 APF0002edl https://www.planetfour.org/subjects/standard/5...

114871 rows × 2 columns

source

normalize_tile_id

def normalize_tile_id(
    tile_id:str, # Full or partial tile ID. If partial, it will be padded with 'APF' and leading zeros.
)->str: # Complete tile ID in format 'APF0000xxx' (always 9 characters)

Normalize a tile ID by adding ‘APF’ prefix and leading zeros if necessary.

assert normalize_tile_id("ci9") == "APF0000ci9"

source

get_subframe

def get_subframe(
    url, # Full subframe image URL.
    download:bool=True, # If False and the image is not already cached, raise instead of
    # attempting a network download (the origin host is offline).
):

Load a subframe image for url from the local pooch cache.

source

is_tile_cached

def is_tile_cached(
    tile_id
):

True if the subframe image for tile_id is already in the local cache.

source

get_url_for_tile

def get_url_for_tile(
    tile_id
):

Deprecated alias for get_url_for_tile_id.

source

get_url_for_tile_id

def get_url_for_tile_id(
    tile_id
):

Return the subframe image URL for tile_id.

tile_id = "APF0000ci9"
tile_id = "1xbr"
get_url_for_tile(tile_id)
/var/folders/y_/4r7grswx4x3dy3s2c0z3h8180000gn/T/ipykernel_15719/2038497295.py:1: DeprecationWarning: get_url_for_tile is deprecated; use get_url_for_tile_id.
  get_url_for_tile(tile_id)
'http://www.planetfour.org/subjects/standard/5143480fea305267e900b75e.jpg'
get_url_for_tile_id("ci9")
'http://www.planetfour.org/subjects/standard/50e7429e5e2ed21240003f51.jpg'

source

get_subframe_for_tile

def get_subframe_for_tile(
    tile_id
):

Deprecated alias for get_subframe_by_tile_id.

source

get_subframe_by_tile_id

def get_subframe_by_tile_id(
    tile_id, download:bool=True
):

Load the subframe image for tile_id from the local cache.

source

get_cached_tile_ids

def get_cached_tile_ids(): # Sorted tile_ids whose subframe image is present in the local cache.

Reverse-map the local subframe cache to the tile_ids available offline.

Scans the pooch tiles cache (pooch.os_cache("p4tools/tiles")), where each file is named <md5(url)>-<subject>.jpg, and maps it back to its tile_id via get_tile_urls. Only files whose name matches pooch.utils.unique_file_name of the looked-up URL are counted, so stray files are ignored.

source

cached_tile_ids_with_markings

def cached_tile_ids_with_markings(
    kind:str='any', # "fan"/"blotch": tiles with at least one fan / blotch.
    # "any": tiles with at least one fan OR blotch (default).
    # "both": tiles with at least one fan AND at least one blotch.
    version:NoneType=None, # Catalog version to check against (defaults to the session version).
): # Sorted cached tile_ids that satisfy `kind`.

Cached tile_ids that have fan and/or blotch markings in the catalog.

source

sample_cached_tile_ids

def sample_cached_tile_ids(
    n:int=1, # Number of tile_ids to draw (without replacement).
    kind:NoneType=None, # If given, restrict to cached tiles that have those markings (see
    # `cached_tile_ids_with_markings`). None (default) samples any cached tile.
    seed:NoneType=None, # Seed for reproducible sampling.
    version:NoneType=None, # Catalog version for the `kind` filter (defaults to the session version).
): # `n` tile_ids randomly drawn from the (optionally filtered) cached set.

Random sample of tile_ids whose subframe image is locally cached.

plt.imshow(get_subframe_by_tile_id(tile_id))

source

get_fans_for_tile

def get_fans_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_fans_for_tile("1xbr").sort_values(by="angle")
obsid marking_id angle distance tile_id image_x image_y n_votes spread version ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
193772 PSP_002942_0935 F02f4ec 36.81 798.87 APF0001xbr 31.62 28633.00 4.0 25.29 1 ... 199.642 0.5 135.831931 -30.228735 215.484939 -3374.391494 -86.310437 -86.353625 97.985477 28
193771 PSP_002942_0935 F02f4eb 37.85 776.30 APF0001xbr 50.80 28635.33 3.0 14.58 1 ... 199.642 0.5 135.831931 -30.220416 215.480259 -3374.391442 -86.310535 -86.353722 97.983479 28
193769 PSP_002942_0935 F02f4e9 40.44 597.17 APF0001xbr 64.76 28626.95 6.0 22.23 1 ... 199.642 0.5 135.831931 -30.217288 215.472891 -3374.391918 -86.310666 -86.353851 97.982932 28
193770 PSP_002942_0935 F02f4ea 40.83 314.90 APF0001xbr 275.16 28732.00 3.0 18.79 1 ... 199.642 0.5 135.831931 -30.103163 215.453344 -3374.393449 -86.311262 -86.354441 97.953881 28

4 rows × 25 columns

get_fans_for_tile("1xbr", version="v1").sort_values(by="angle")
/var/folders/y_/4r7grswx4x3dy3s2c0z3h8180000gn/T/ipykernel_15719/1519352468.py:1: UserWarning: Tile APF0001xbr is not in the v1 catalog — its observation was not included in this release. Try a different version.
  get_fans_for_tile("1xbr", version="v1").sort_values(by="angle")
obsid marking_id angle distance tile_id image_x image_y n_votes spread version ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY

0 rows × 25 columns

source

get_blotches_for_tile

def get_blotches_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_blotches_for_tile(tile_id, version="v3")
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY
323107 PSP_002942_0935 B04ee23 43.15 APF0001xbr 101.57 28868.46 7.0 29.92 16.73 0.69 ... 199.642 0.5 135.831931 -30.132724 215.557621 -3374.38611 -86.309439 -86.352639 97.95779 28

1 rows × 24 columns

get_blotches_for_tile(tile_id, version="v1")
/var/folders/y_/4r7grswx4x3dy3s2c0z3h8180000gn/T/ipykernel_15719/27976457.py:1: UserWarning: Tile APF0001xbr is not in the v1 catalog — its observation was not included in this release. Try a different version.
  get_blotches_for_tile(tile_id, version="v1")
obsid marking_id angle tile_id image_x image_y n_votes radius_1 radius_2 vote_ratio ... l_s map_scale north_azimuth BodyFixedCoordinateX BodyFixedCoordinateY BodyFixedCoordinateZ PlanetocentricLatitude PlanetographicLatitude Longitude MY

0 rows × 24 columns

source

get_hirise_id_for_tile

def get_hirise_id_for_tile(
    tile_id, version:NoneType=None
):

Call self as a function.

get_hirise_id_for_tile(tile_id)
'PSP_002942_0935'
# set_catalog_version changes the module default
assert _DEFAULT_VERSION == "v3.1"
set_catalog_version("v3")
assert _DEFAULT_VERSION == "v3"
set_catalog_version("v3.1")  # restore
assert _DEFAULT_VERSION == "v3.1"
# catalog_version context manager temporarily overrides and restores the default
assert _DEFAULT_VERSION == "v3.1"
with catalog_version("v1"):
    assert _DEFAULT_VERSION == "v1"
assert _DEFAULT_VERSION == "v3.1"  # restored after context exit
# catalog_version context manager restores even on exception
try:
    with catalog_version("v3"):
        assert _DEFAULT_VERSION == "v3"
        raise RuntimeError("simulated error")
except RuntimeError:
    pass
assert _DEFAULT_VERSION == "v3.1"  # restored despite exception
# _resolve_version returns explicit version when given, default otherwise, and normalises .0 suffixes
assert _resolve_version("v1") == "v1"
assert _resolve_version("v1.0") == "v1"
assert _resolve_version("v3.0") == "v3"
assert _resolve_version("v3.1") == "v3.1"
assert _resolve_version(None) == "v3.1"

Convenience joiners

Tiny helpers that left-join an obsid-keyed dataframe with metafull (MY, L_s) or get_region_names (roi_name).

source

attach_roi

def attach_roi(
    df, obsid_col:str='obsid', version:str='v3.1'
):

Left-join df with get_region_names(version=...)[obsid, roi_name].

source

attach_my

def attach_my(
    df, obsid_col:str='obsid', version:str='v3.1'
):

Left-join df with metafull[OBSERVATION_ID, MY, SOLAR_LONGITUDE].

The metafull columns are renamed to obsid, MY, l_s.