IO module

This module takes care of receiving the data catalogs.

postprocessor

 postprocessor (fname, action, pup)

*Post-processing hook to process the downloaded v1 catalog file.

Postprocessing steps are: 1. unzip to CSV 2. load CSV into pandas 3. save as parquet file.

This time, at the next read, the performance is much increased due to reading the parquet file instead of loading the CSV file every time.*

	Type	Details
fname	str	Full path of the zipped file in local storage
action	str	One of “download” (file doesn’t exist and will download), “update” (file is outdated and will download), and “fetch” (file exists and is updated so no download).
pup	Pooch	The instance of Pooch that called the processor function.
Returns	str	The full path to the unzipped file. (Return the same fname is your processor doesn’t modify the file).

v1.fetch("fans", **kwargs)

Downloading file 'fans' from 'doi:10.5281/zenodo.8102805/P4_catalog_v1.1_L1C_cut_0.5_fan.csv.zip' to '/Users/maye/Library/Caches/p4tools/v1'.
100%|█████████████████████████████████████| 9.94M/9.94M [00:00<00:00, 9.88GB/s]
Unzipping contents of '/Users/maye/Library/Caches/p4tools/v1/fans' to '/Users/maye/Library/Caches/p4tools/v1/fans.unzip'
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v1/fans.unzip/P4_catalog_v1.1_L1C_cut_0.5_fan.parq.

PosixPath('/Users/maye/Library/Caches/p4tools/v1/fans.unzip/P4_catalog_v1.1_L1C_cut_0.5_fan.parq')

v1.fetch("blotches", **kwargs)

Downloading file 'blotches' from 'doi:10.5281/zenodo.8102805/P4_catalog_v1.1_L1C_cut_0.5_blotch.csv.zip' to '/Users/maye/Library/Caches/p4tools/v1'.
100%|█████████████████████████████████████| 15.2M/15.2M [00:00<00:00, 7.39GB/s]
Unzipping contents of '/Users/maye/Library/Caches/p4tools/v1/blotches' to '/Users/maye/Library/Caches/p4tools/v1/blotches.unzip'
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v1/blotches.unzip/P4_catalog_v1.1_L1C_cut_0.5_blotch.parq.

PosixPath('/Users/maye/Library/Caches/p4tools/v1/blotches.unzip/P4_catalog_v1.1_L1C_cut_0.5_blotch.parq')

pd.read_parquet(v1.fetch('metadata', **kwargs))

Downloading file 'metadata' from 'doi:10.5281/zenodo.8102805/P4_catalog_v1.1_metadata.csv.zip' to '/Users/maye/Library/Caches/p4tools/v1'.
100%|█████████████████████████████████████| 6.99k/6.99k [00:00<00:00, 7.18MB/s]
Unzipping contents of '/Users/maye/Library/Caches/p4tools/v1/metadata' to '/Users/maye/Library/Caches/p4tools/v1/metadata.unzip'
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v1/metadata.unzip/P4_catalog_v1.1_metadata.parq.

	OBSERVATION_ID	IMAGE_CENTER_LATITUDE	IMAGE_CENTER_LONGITUDE	SOLAR_LONGITUDE	START_TIME	map_scale	north_azimuth	# of tiles
0	ESP_011296_0975	-82.1965	225.2530	178.833	2008-12-23 16:15:26	1.00	110.600107	91
1	ESP_011341_0980	-81.7969	76.1304	180.809	2008-12-27 04:25:02	0.50	110.208923	126
2	ESP_011348_0950	-85.0427	259.0940	181.117	2008-12-27 17:29:17	1.00	123.624057	91
3	ESP_011350_0945	-85.2160	181.4150	181.205	2008-12-27 21:14:01	0.50	99.672793	126
4	ESP_011351_0945	-85.2157	181.5480	181.249	2008-12-27 23:05:54	1.00	127.960688	91
...	...	...	...	...	...	...	...	...
216	ESP_022273_0950	-84.8184	65.7963	282.247	2011-04-28 00:21:01	0.25	117.236648	270
217	ESP_022339_0935	-86.2751	99.6123	285.423	2011-05-03 03:45:55	0.25	153.768164	315
218	ESP_022379_0930	-87.0045	86.5510	287.340	2011-05-06 06:34:07	0.25	153.003708	260
219	ESP_022510_0980	-81.6988	66.3188	293.572	2011-05-16 11:34:43	0.50	107.877998	72
220	ESP_022699_0985	-81.4043	295.7820	302.421	2011-05-31 05:01:48	0.50	96.757068	90

221 rows × 8 columns

v3.fetch("fans", **kwargs)

Downloading file 'fans' from 'https://refubium.fu-berlin.de/bitstream/handle/fub188/46287/P4_catalog_Full_Release_v3.0_L1C_cut_0.5_fan_meta_merged.csv' to '/Users/maye/Library/Caches/p4tools/v3'.
0.00B [00:00, ?B/s]     
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v3/fans.parq.

PosixPath('/Users/maye/Library/Caches/p4tools/v3/fans.parq')

v3.fetch("blotches", **kwargs)

Downloading file 'blotches' from 'https://refubium.fu-berlin.de/bitstream/handle/fub188/46287/P4_catalog_Full_Release_v3.0_L1C_cut_0.5_blotch_meta_merged.csv' to '/Users/maye/Library/Caches/p4tools/v3'.
0.00B [00:00, ?B/s]     
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v3/blotches.parq.

PosixPath('/Users/maye/Library/Caches/p4tools/v3/blotches.parq')

source

get_tile_urls

 get_tile_urls ()

only v1 exists

source

get_region_names

 get_region_names ()

only v1 exists

source

get_meta_data

 get_meta_data (version='v3')

source

get_tile_coords

 get_tile_coords (version='v3')

source

get_fan_catalog

 get_fan_catalog (version='v3')

source

get_blotch_catalog

 get_blotch_catalog (version='v3')

get_meta_data()

Downloading file 'metadata' from 'https://refubium.fu-berlin.de/bitstream/handle/fub188/46287/P4_catalog_Full_Release_v3.0_metadata.csv' to '/Users/maye/Library/Caches/p4tools/v3'.
0.00B [00:00, ?B/s]
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v3/metadata.parq.

	OBSERVATION_ID	IMAGE_CENTER_LATITUDE	IMAGE_CENTER_LONGITUDE	SOLAR_LONGITUDE	START_TIME	map_scale	north_azimuth	# of tiles
0	PSP_001916_1105	-69.0451	350.7460	155.049	2006-12-23 17:32:35	1.00	98.778615	90
1	PSP_001920_1110	-68.9785	241.7860	155.212	2006-12-24 01:01:22	1.00	99.072922	69
2	PSP_001929_1105	-69.2528	355.9350	155.577	2006-12-24 17:51:13	1.00	98.871037	144
3	PSP_002081_1055	-74.0197	168.6750	161.817	2007-01-05 14:05:19	1.00	100.672940	109
4	PSP_002147_1055	-74.1610	168.6620	164.567	2007-01-10 17:30:57	1.00	102.328402	109
...	...	...	...	...	...	...	...	...
462	ESP_047653_0950	-85.0542	180.2900	229.690	2016-09-25 14:35:20	0.25	119.836395	455
463	ESP_047696_0935	-86.3881	99.0205	231.795	2016-09-28 23:00:27	0.50	131.679872	62
464	ESP_048431_0950	-85.0536	180.2880	268.059	2016-11-25 05:46:04	0.25	123.242307	455
465	ESP_048576_0950	-85.0559	180.3070	275.155	2016-12-06 12:58:38	0.25	122.458764	455
466	ESP_049209_0950	-85.1548	180.7940	305.236	2017-01-24 20:49:23	0.25	125.301869	455

467 rows × 8 columns

get_blotch_catalog('v1').tail()

	marking_id	angle	tile_id	image_x	image_y	n_votes	obsid	radius_1	radius_2	vote_ratio	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude
249796	B044be7	0.00	APF0000idg	1276.20	14257.20	5	ESP_022699_0985	10.00	10.0	1.0	...	0.00	302.421	0.5	96.757068	219.906430	-455.912914	-3342.551722	-81.388876	-81.488458	295.750015
249797	B044be8	0.00	APF0000idg	1311.25	14345.75	4	ESP_022699_0985	10.00	10.0	1.0	...	0.00	302.421	0.5	96.757068	219.897692	-455.963924	-3342.536671	-81.388131	-81.487722	295.746616
249798	B044be9	0.00	APF0000idg	1266.00	14355.25	4	ESP_022699_0985	10.00	10.0	1.0	...	0.00	302.421	0.5	96.757068	219.925196	-455.960274	-3342.537242	-81.387987	-81.487580	295.749599
249799	B044bea	168.67	APF0000idh	74.07	148.86	7	ESP_022699_0985	16.84	14.1	1.0	...	0.42	302.421	0.5	96.757068	218.194755	-448.614747	-3343.589608	-81.514080	-81.612257	295.937105
249800	B044beb	172.85	APF0000idh	265.00	98.33	3	ESP_022699_0985	40.55	25.6	1.0	...	0.12	302.421	0.5	96.757068	218.086509	-448.624657	-3343.607336	-81.514768	-81.612938	295.925426

5 rows × 23 columns

get_fan_catalog()

	marking_id	angle	distance	tile_id	image_x	image_y	n_votes	obsid	spread	version	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude
0	F000000	244.78	44.03	APF00006nn	717.45	20631.10	10.0	ESP_011296_0975	57.33	1	...	-0.88	178.833	1.00	110.600008	-319.361873	-324.061392	-3350.426985	-82.266657	-82.356346	225.418477
1	F000001	213.33	51.03	APF00006nn	95.11	20612.33	9.0	ESP_011296_0975	50.71	1	...	-0.55	178.833	1.00	110.600008	-319.084076	-324.615071	-3350.400057	-82.263241	-82.352970	225.492303
2	F000002	212.13	47.47	APF00006nn	194.20	20486.20	10.0	ESP_011296_0975	39.58	1	...	-0.53	178.833	1.00	110.600008	-319.012600	-324.473908	-3350.420251	-82.265819	-82.355518	225.486261
3	F000003	224.17	36.21	APF00006nn	207.94	20561.88	8.0	ESP_011296_0975	54.17	1	...	-0.69	178.833	1.00	110.600008	-319.086116	-324.492548	-3350.411302	-82.264710	-82.354422	225.481306
4	F000004	213.32	36.20	APF00006nn	450.17	20562.83	6.0	ESP_011296_0975	77.23	1	...	-0.55	178.833	1.00	110.600008	-319.188599	-324.274500	-3350.423477	-82.266141	-82.355836	225.452854
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
273416	F042c08	116.85	127.63	APF0001dwc	1313.78	50940.00	9.0	PSP_005119_0950	41.27	1	...	0.89	304.812	0.25	137.094609	130.306925	280.888054	-3367.349516	-84.746191	-84.807518	65.112918
273417	F042c09	82.75	87.79	APF0001dqg	775.65	54522.54	4.0	PSP_005119_0950	39.21	2	...	0.99	304.812	0.25	137.094609	131.118946	281.330260	-3367.271243	-84.733527	-84.795000	65.011294
273418	F042c0a	292.80	71.73	APF0001gyy	1177.36	114102.00	3.0	PSP_005513_0950	53.26	1	...	-0.92	322.536	0.50	122.906026	-14.219321	315.811256	-3366.400837	-84.635214	-84.697822	92.577987
273419	F042c0b	61.74	210.17	APF0001yca	2561.92	31264.67	6.0	PSP_005579_0935	24.45	1	...	0.88	325.418	0.25	129.651195	-33.403880	210.012552	-3374.827184	-86.394481	-86.436690	99.037565
273420	F042c0c	286.52	100.58	APF0002k5s	512.84	6617.01	3.0	PSP_005619_0945	26.36	2	...	-0.96	327.153	0.50	125.245088	24.012192	259.220326	-3371.710023	-85.584946	-85.636566	84.707659

273421 rows × 24 columns

get_region_names()

Downloading file 'region_names' from 'doi:10.5281/zenodo.8102805/region_names.zip' to '/Users/maye/Library/Caches/p4tools/v1'.
100%|████████████████████████████████████████| 786k/786k [00:00<00:00, 632MB/s]
Unzipping contents of '/Users/maye/Library/Caches/p4tools/v1/region_names' to '/Users/maye/Library/Caches/p4tools/v1/region_names.unzip'
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v1/region_names.unzip/region_names.parq.

	obsid	lat_IND	lon_IND	roi_name	minimal_distance	lat_WORD	lon_WORD	time	MY
0	ESP_012079_0945	-85.4074	103.970	Macclesfield	0.944588	-85.401	103.901	2009-02-22 16:32:19	29
1	ESP_021494_0945	-85.4041	103.943	Macclesfield	0.511133	-85.401	103.901	2011-02-26 07:30:27	30
2	ESP_011407_0945	-85.4065	103.983	Macclesfield	0.957066	-85.401	103.901	2009-01-01 07:49:15	29
3	ESP_020782_0945	-85.4071	103.967	Macclesfield	0.901696	-85.401	103.901	2011-01-01 19:57:51	30
4	ESP_020716_0945	-85.4086	103.979	Macclesfield	1.098928	-85.401	103.901	2010-12-27 16:32:24	30
...	...	...	...	...	...	...	...	...	...
449	ESP_029762_0940	-85.7567	106.026	Manhattan2	0.782410	-85.751	105.971	2012-12-01 13:16:26	31
450	ESP_037977_0940	-85.6793	105.614	Manhattan2	8.544184	-85.751	105.971	2014-09-02 16:34:56	32
451	ESP_037976_0940	-85.6694	105.903	Manhattan2	9.131402	-85.751	105.971	2014-09-02 14:43:05	32
452	ESP_040311_0940	-85.7806	106.025	Manhattan2	3.335817	-85.751	105.971	2015-03-03 12:35:10	32
453	ESP_040193_0940	-85.7860	106.143	Manhattan2	4.158145	-85.751	105.971	2015-02-22 07:54:37	32

454 rows × 9 columns

get_tile_urls()

Downloading file 'tile_urls' from 'doi:10.5281/zenodo.8102805/tile_urls.csv.zip' to '/Users/maye/Library/Caches/p4tools/v1'.
100%|████████████████████████████████████████| 771k/771k [00:00<00:00, 348MB/s]
Unzipping contents of '/Users/maye/Library/Caches/p4tools/v1/tile_urls' to '/Users/maye/Library/Caches/p4tools/v1/tile_urls.unzip'
Converting CSV to parquet and storing as /Users/maye/Library/Caches/p4tools/v1/tile_urls.unzip/tile_urls.parq.

	tile_id	tile_url
0	APF0000coq	http://www.planetfour.org/subjects/standard/50...
1	APF0000cro	http://www.planetfour.org/subjects/standard/50...
2	APF0000cs6	http://www.planetfour.org/subjects/standard/50...
3	APF0000co1	http://www.planetfour.org/subjects/standard/50...
4	APF0000cpd	http://www.planetfour.org/subjects/standard/50...
...	...	...
114866	APF0002eep	https://www.planetfour.org/subjects/standard/5...
114867	APF0002ee9	https://www.planetfour.org/subjects/standard/5...
114868	APF0002eej	https://www.planetfour.org/subjects/standard/5...
114869	APF0002ee0	https://www.planetfour.org/subjects/standard/5...
114870	APF0002edl	https://www.planetfour.org/subjects/standard/5...

114871 rows × 2 columns

source

normalize_tile_id

 normalize_tile_id (tile_id:str)

Normalize a tile ID by adding ‘APF’ prefix and leading zeros if necessary.

	Type	Details
tile_id	str	Full or partial tile ID. If partial, it will be padded with ‘APF’ and leading zeros.
Returns	str	Complete tile ID in format ‘APF0000xxx’ (always 9 characters)

assert normalize_tile_id("ci9") == "APF0000ci9"

source

get_subframe

 get_subframe (url)

source

get_url_for_tile

 get_url_for_tile (tile_id)

source

get_url_for_tile_id

 get_url_for_tile_id (tile_id)

tile_id = "APF0000ci9"
tile_id = '1xbr'

get_url_for_tile(tile_id)

'http://www.planetfour.org/subjects/standard/5143480fea305267e900b75e.jpg'

get_url_for_tile_id("ci9")

'http://www.planetfour.org/subjects/standard/50e7429e5e2ed21240003f51.jpg'

source

get_subframe_for_tile

 get_subframe_for_tile (tile_id)

source

get_subframe_by_tile_id

 get_subframe_by_tile_id (tile_id)

plt.imshow(get_subframe_for_tile(tile_id))

source

get_fans_for_tile

 get_fans_for_tile (tile_id, version='v3')

get_fans_for_tile('1xbr').sort_values(by='angle')

	marking_id	angle	distance	tile_id	image_x	image_y	n_votes	obsid	spread	version	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude
201216	F031200	36.81	798.87	APF0001xbr	31.62	28633.00	4.0	PSP_002942_0935	25.29	1	...	0.60	199.642	0.5	135.831931	-30.228735	215.484939	-3374.391494	-86.310437	-86.353625	97.985477
201217	F031201	37.85	776.30	APF0001xbr	50.80	28635.33	3.0	PSP_002942_0935	14.58	1	...	0.61	199.642	0.5	135.831931	-30.220416	215.480259	-3374.391442	-86.310535	-86.353722	97.983479
201215	F0311ff	40.44	597.17	APF0001xbr	64.76	28626.95	6.0	PSP_002942_0935	22.23	1	...	0.65	199.642	0.5	135.831931	-30.217288	215.472891	-3374.391918	-86.310666	-86.353851	97.982932
201214	F0311fe	40.83	314.90	APF0001xbr	275.16	28732.00	3.0	PSP_002942_0935	18.79	1	...	0.65	199.642	0.5	135.831931	-30.103163	215.453344	-3374.393449	-86.311262	-86.354441	97.953881

4 rows × 24 columns

get_fans_for_tile('1xbr', version='v1').sort_values(by='angle')

	marking_id	angle	distance	tile_id	image_x	image_y	n_votes	obsid	spread	version	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude

0 rows × 24 columns

source

get_blotches_for_tile

 get_blotches_for_tile (tile_id, version='v3')

get_blotches_for_tile(tile_id, version='v3')

	marking_id	angle	tile_id	image_x	image_y	n_votes	obsid	radius_1	radius_2	vote_ratio	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude
323107	B04ee23	43.15	APF0001xbr	101.57	28868.46	7.0	PSP_002942_0935	29.92	16.73	0.69	...	0.67	199.642	0.5	135.831931	-30.132724	215.557621	-3374.38611	-86.309439	-86.352639	97.95779

1 rows × 23 columns

get_blotches_for_tile(tile_id, version='v1')

	marking_id	angle	tile_id	image_x	image_y	n_votes	obsid	radius_1	radius_2	vote_ratio	...	y_angle	l_s	map_scale	north_azimuth	BodyFixedCoordinateX	BodyFixedCoordinateY	BodyFixedCoordinateZ	PlanetocentricLatitude	PlanetographicLatitude	Longitude

0 rows × 23 columns

source

get_hirise_id_for_tile

 get_hirise_id_for_tile (tile_id, version='v3')

get_hirise_id_for_tile(tile_id)

'PSP_002942_0935'