Update Web Layers¶

In [1]:

# import the GIS class in gis module
from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection
from time import strftime
import os, requests, zipfile
import pandas as pd
import geopandas as gpd
import arcpy,datetime,json

# import the GIS class in gis module from arcgis.gis import GIS from arcgis.features import FeatureLayerCollection from time import strftime import os, requests, zipfile import pandas as pd import geopandas as gpd import arcpy,datetime,json

Download data¶

Steps:¶

1. Define Feature Layer ID:

   • Set the ID of the feature layer from which the data will be downloaded.

2. Specify Download URL:

   • Construct the download URL using a formatted string.
   • The URL points to the location of the file to be downloaded.

3. Specify Download Path:

   • Define the absolute path where the downloaded file will be saved.

4. Download the File:

   • Use the requests library to send a GET request to the download URL.
   • Check if the request was successful (status code 200).

5. Save the File:

   • If the request was successful, open the file in binary write mode and write the content to it.
   • Print the path where the file was saved if the download was successful.

In [2]:

feature_layer_id = "0fb67f50b99e4979a9fc49da11c9148e"
# download_url = f"https://www.pasda.psu.edu/download/dep/StorageTankLocations_Active{strftime('%Y')}_{strftime('%m')}.zip"
download_url = f"https://www.pasda.psu.edu/download/dep/StorageTankLocations_Active2024_05.zip"
download_path = os.path.abspath("./assets")

# Filename from the URL
filename = download_url.split('/')[-1]

# Full path to save the file
file_path = os.path.join(download_path, filename)

# Download the file
response = requests.get(download_url)

# Check if the request was successful
if response.status_code == 200:
    # Write content to the file
    with open(file_path, 'wb') as file:
        file.write(response.content)
    print(f"File downloaded successfully to: {file_path}")
else:
    print("Failed to download the file.")

feature_layer_id = "0fb67f50b99e4979a9fc49da11c9148e" # download_url = f"https://www.pasda.psu.edu/download/dep/StorageTankLocations_Active{strftime('%Y')}_{strftime('%m')}.zip" download_url = f"https://www.pasda.psu.edu/download/dep/StorageTankLocations_Active2024_05.zip" download_path = os.path.abspath("./assets") # Filename from the URL filename = download_url.split('/')[-1] # Full path to save the file file_path = os.path.join(download_path, filename) # Download the file response = requests.get(download_url) # Check if the request was successful if response.status_code == 200: # Write content to the file with open(file_path, 'wb') as file: file.write(response.content) print(f"File downloaded successfully to: {file_path}") else: print("Failed to download the file.")

File downloaded successfully to: d:\Eric-Personal\python\docs\Geoprocessing\assets\StorageTankLocations_Active2024_05.zip

unzip downloaded data¶

In [3]:

# Extract the downloaded ZIP file
with zipfile.ZipFile(file_path, 'r') as zip_ref:
    zip_ref.extractall(download_path)

# Path to the extracted CSV file
shp_file_path = os.path.join(download_path, "StorageTankLocations_Active2024_05.shp")

# Read the downloaded CSV data
downloaded_data = gpd.read_file(shp_file_path)
print(downloaded_data.shape)
downloaded_data.head()

# Extract the downloaded ZIP file with zipfile.ZipFile(file_path, 'r') as zip_ref: zip_ref.extractall(download_path) # Path to the extracted CSV file shp_file_path = os.path.join(download_path, "StorageTankLocations_Active2024_05.shp") # Read the downloaded CSV data downloaded_data = gpd.read_file(shp_file_path) print(downloaded_data.shape) downloaded_data.head()

(11289, 25)

Out[3]:

	FACILITY_I	FACILITY_N	FACILITY_A	FACILITY_1	FACILITY_C	FACILITY_S	FACILITY_Z	COUNTY_ID	FACILITY_2	MUNICIPALI	...	TANK_OWN_3	TANK_OWN_4	TANK_OWN_5	TANK_OWN_6	PRIMARY_FA	SITE_ID	LATITUDE	LONGITUDE	TANK_INFOR	geometry
0	56-39595	CHEMSTREAM	NORTH STAR IND PARK COMMERCE DR	None	STOYSTOWN	PA	15563	2463	Somerset	2130	...	None	HOMER CITY	PA	15748-1422	695172	691484	40.137605	-79.025093	http://cedatareporting.pa.gov/Reportserver/Pag...	POINT (-8797033.113 4885958.858)
1	35-52629	THROOP DIVERSION CHAMBER	862 ENTERPRISE ST	None	DICKSON CITY	PA	18519	2442	Lackawanna	1403	...	None	OLYPHANT	PA	18447-0280	695938	645882	41.461955	-75.616175	http://cedatareporting.pa.gov/Reportserver/Pag...	POINT (-8417554.097 5080720.569)
2	11-39625	MARTIN GENERAL STORE 230	4057 ADMIRAL PEARY HWY	None	EBENSBURG	PA	15931-3919	2418	Cambria	570	...	None	BELLWOOD	PA	16617-1922	696735	693483	40.460940	-78.769425	http://cedatareporting.pa.gov/Reportserver/Pag...	POINT (-8768572.281 4933152.658)
3	15-39634	FREEDOM VILLAGE AT BRANDYWINE	15 FREEDOM BLVD	None	WEST BRANDYWINE	PA	19320-1549	2422	Chester	753	...	None	WEST BRANDYWINE	PA	19320-1549	697311	694230	40.013371	-75.790556	http://cedatareporting.pa.gov/Reportserver/Pag...	POINT (-8436966.101 4867885.507)
4	45-39639	ROCK HILL CONCRETE	UPPER CHERRY VALLEY ROAD	RD4 BOX 140	STROUDSBURG	PA	18360	2452	Monroe	1781	...	None	DOWNERS GROVE	IL	60515-5560	697481	694449	40.927418	-75.271720	http://cedatareporting.pa.gov/Reportserver/Pag...	POINT (-8379209.542 5001641.723)

5 rows × 25 columns

Log into ArcGIS Online¶

In [4]:

gis = GIS("home")
print("Logged in as " + str(gis.properties.user.username))

gis = GIS("home") print("Logged in as " + str(gis.properties.user.username))

Logged in as yxy180050@utdallas.edu_UTDEPPS

In [5]:

# Get the existing feature layer data

data_item = gis.content.get(itemid=feature_layer_id)
feature_layer = data_item.layers[0]

# Query all data from the existing feature layer including geometry
existing_data = pd.DataFrame.spatial.from_layer(feature_layer)
print(existing_data.shape)
existing_data.head()

# Get the existing feature layer data data_item = gis.content.get(itemid=feature_layer_id) feature_layer = data_item.layers[0] # Query all data from the existing feature layer including geometry existing_data = pd.DataFrame.spatial.from_layer(feature_layer) print(existing_data.shape) existing_data.head()

(11289, 26)

Out[5]:

	COUNTY_ID	FACILITY_1	FACILITY_2	FACILITY_A	FACILITY_C	FACILITY_I	FACILITY_M	FACILITY_N	FACILITY_S	FACILITY_Z	...	SHAPE	SITE_ID	TANK_INFOR	TANK_OWNER	TANK_OWN_1	TANK_OWN_2	TANK_OWN_3	TANK_OWN_4	TANK_OWN_5	TANK_OWN_6
0	2409		Allegheny	2000 S FAYETTE WAY	MCDONALD	02-97992	South Fayette Twp	SOUTHERN BELTWAY MAINT	PA	15057	...	{"x": -8926798.6883, "y": 4915484.524700001, "...	849719	http://cedatareporting.pa.gov/Reportserver/Pag...	55964	PA TURNPIKE COMM	PO BOX 67676		HARRISBURG	PA	17106-7676
1	2409		Allegheny	5993 S BRADDOCK AVE	EDGEWOOD	02-55204	Edgewood Boro	GET GO 3147	PA	15218	...	{"x": -8893547.8904, "y": 4928583.006300002, "...	828738	http://cedatareporting.pa.gov/Reportserver/Pag...	125094	GIANT EAGLE INC	101 KAPPA DR		PITTSBURGH	PA	15238
2	2443		Lancaster	520 LIME QUARRY RD	GAP	36-09911	Salisbury Twp	LIMEVILLE QUARRY	PA	17527-9739	...	{"x": -8457076.5235, "y": 4867733.767300002, "...	577229	http://cedatareporting.pa.gov/Reportserver/Pag...	62856	NEW ENTERPRISE STONE & LIME CO INC	3912 BRUMBAUGH RD		NEW ENTERPRISE	PA	16664-9137
3	2409		Allegheny	3901 CALIFORNIA AVE	PITTSBURGH	02-55599	Pittsburgh City	CALIFORNIA MINI MART	PA	15212-1610	...	{"x": -8910012.0431, "y": 4935795.365699999, "...	842194	http://cedatareporting.pa.gov/Reportserver/Pag...	350369	CALIFORNIA MINI MART LLC	3901 CALIFORNIA AVE		PITTSBURGH	PA	15212-1610
4	2429	HANGAR 604	Dauphin	604 AIRPORT DR	MIDDLETOWN	22-64095	Lower Swatara Twp	MIWD FLIGHT HANGER 604	PA	17057-5036	...	{"x": -8545898.0112, "y": 4894937.820699997, "...	572910	http://cedatareporting.pa.gov/Reportserver/Pag...	357377	MIWD FLIGHT LLC	650 W MARKET ST		GRATZ	PA	17030-9701

5 rows × 26 columns

Make a backup Layer¶

Steps:¶

1. Check for Existing Feature Layer:

   • Search for existing feature layers with the specified name using gis.content.search().
   • If any feature layers with the same name are found, delete them using the delete() method.

2. Create New Feature Layer:

   • Convert existing spatial data to a feature layer using spatial.to_featurelayer().
   • Specify the name of the new feature layer and optionally the folder where it will be saved.

3. Handle Exceptions:

   • Catch any exceptions that occur during the process and print error messages.

In [6]:

try:
    # Check if a feature layer with the same name already exists
    new_feature_layer_name = "Backup_FeatureLayer"
    existing_layers = gis.content.search(query=new_feature_layer_name, item_type="Feature Service")
    
    # If a feature layer with the same name exists, delete it
    for item in existing_layers:
        item.delete()
    
    # Create a new hosted feature layer
    new_feature_layer_item = existing_data.spatial.to_featurelayer(new_feature_layer_name, folder = "ArcGIS Python API")
    print("Created the new feature layer service successfully.")
    
except Exception as e:
    print(f"Error creating the new feature layer: {e}")

try: # Check if a feature layer with the same name already exists new_feature_layer_name = "Backup_FeatureLayer" existing_layers = gis.content.search(query=new_feature_layer_name, item_type="Feature Service") # If a feature layer with the same name exists, delete it for item in existing_layers: item.delete() # Create a new hosted feature layer new_feature_layer_item = existing_data.spatial.to_featurelayer(new_feature_layer_name, folder = "ArcGIS Python API") print("Created the new feature layer service successfully.") except Exception as e: print(f"Error creating the new feature layer: {e}")

Created the new feature layer service successfully.

Copy everything in the existing layer to the backup layer¶

In [7]:

new_feature_layer_item

new_feature_layer_item

Out[7]:

Backup_FeatureLayer
Feature Layer Collection by yxy180050@utdallas.edu_UTDEPPS
Last Modified: June 05, 2024
0 comments, 0 views

In [ ]:

# # Define the schema for the new layer based on the existing layer
# fields = [{"name": field['name'], "type": field['type'], "alias": field['alias']} for field in feature_layer.properties.fields]
# geometry_type = feature_layer.properties.geometryType

# layer_definition = {
#     "layers": [{
#         "name": "BackupLayer",
#         "geometryType": geometry_type,
#         "fields": fields
#     }]
# }

# # Add the layer definition to the new feature layer collection
# new_feature_layer_collection.manager.add_to_definition(layer_definition)

# # Define the schema for the new layer based on the existing layer # fields = [{"name": field['name'], "type": field['type'], "alias": field['alias']} for field in feature_layer.properties.fields] # geometry_type = feature_layer.properties.geometryType # layer_definition = { # "layers": [{ # "name": "BackupLayer", # "geometryType": geometry_type, # "fields": fields # }] # } # # Add the layer definition to the new feature layer collection # new_feature_layer_collection.manager.add_to_definition(layer_definition)

In [ ]:

# # Convert the DataFrame to a list of dictionaries
# features = existing_data.spatial.to_featureset().features

# # Define a function to chunk data
# def chunk_data(data, chunk_size):
#     for i in range(0, len(data), chunk_size):
#         yield data[i:i + chunk_size]

# # Set the chunk size
# chunk_size = 100  # Adjust the chunk size as necessary

# try:
#     # Get the layer in the new feature layer collection
#     new_layer = new_feature_layer_collection.layers[0]

#     # Add features in chunks
#     for chunk in chunk_data(features, chunk_size):
#         new_layer.edit_features(adds=chunk)
#     print("Successfully added all features to the new layer.")

# except Exception as e:
#     print(f"Error: {e}")

# # Convert the DataFrame to a list of dictionaries # features = existing_data.spatial.to_featureset().features # # Define a function to chunk data # def chunk_data(data, chunk_size): # for i in range(0, len(data), chunk_size): # yield data[i:i + chunk_size] # # Set the chunk size # chunk_size = 100 # Adjust the chunk size as necessary # try: # # Get the layer in the new feature layer collection # new_layer = new_feature_layer_collection.layers[0] # # Add features in chunks # for chunk in chunk_data(features, chunk_size): # new_layer.edit_features(adds=chunk) # print("Successfully added all features to the new layer.") # except Exception as e: # print(f"Error: {e}")

In [ ]:

# features[0]

# features[0]

Check the difference between exisitng data and downloaded data¶

In [8]:

# Convert column names to lowercase for comparison
existing_columns= existing_data.columns
downloaded_columns = downloaded_data.columns

# Find columns in downloaded_data that are present in existing_data
checking_col = downloaded_columns[downloaded_columns.isin(existing_columns)]

checking_col

# Convert column names to lowercase for comparison existing_columns= existing_data.columns downloaded_columns = downloaded_data.columns # Find columns in downloaded_data that are present in existing_data checking_col = downloaded_columns[downloaded_columns.isin(existing_columns)] checking_col

Out[8]:

Index(['FACILITY_I', 'FACILITY_N', 'FACILITY_A', 'FACILITY_1', 'FACILITY_C',
       'FACILITY_S', 'FACILITY_Z', 'COUNTY_ID', 'FACILITY_2', 'MUNICIPALI',
       'FACILITY_M', 'REG_EXPIRA', 'TANK_OWNER', 'TANK_OWN_1', 'TANK_OWN_2',
       'TANK_OWN_3', 'TANK_OWN_4', 'TANK_OWN_5', 'TANK_OWN_6', 'PRIMARY_FA',
       'SITE_ID', 'LATITUDE', 'LONGITUDE', 'TANK_INFOR'],
      dtype='object')

In [9]:

# Identify new rows in the downloaded data
new_rows = downloaded_data[~downloaded_data[checking_col].apply(tuple, axis=1).isin(existing_data[checking_col].apply(tuple, axis=1))].dropna()
new_rows.shape

# Identify new rows in the downloaded data new_rows = downloaded_data[~downloaded_data[checking_col].apply(tuple, axis=1).isin(existing_data[checking_col].apply(tuple, axis=1))].dropna() new_rows.shape

Out[9]:

(35, 25)

In [10]:

new_rows.index

new_rows.index

Out[10]:

Index([  290,   291,   627,   889,  1013,  1654,  1680,  1745,  1900,  3308,
        3475,  3698,  4309,  4807,  5788,  6799,  7044,  7201,  7577,  7620,
        7985,  8038,  9042,  9221,  9239,  9485,  9651,  9798, 10631, 10712,
       10769, 10798, 10889, 10991, 11175],
      dtype='int64')

Overwrite features¶

Make a backup Layer¶

In [11]:

# Append new rows to the existing data
combined_data = pd.concat([existing_data, new_rows], ignore_index=True)
print(combined_data.shape)
combined_data.head()

# Append new rows to the existing data combined_data = pd.concat([existing_data, new_rows], ignore_index=True) print(combined_data.shape) combined_data.head()

(11324, 27)

Out[11]:

	COUNTY_ID	FACILITY_1	FACILITY_2	FACILITY_A	FACILITY_C	FACILITY_I	FACILITY_M	FACILITY_N	FACILITY_S	FACILITY_Z	...	SITE_ID	TANK_INFOR	TANK_OWNER	TANK_OWN_1	TANK_OWN_2	TANK_OWN_3	TANK_OWN_4	TANK_OWN_5	TANK_OWN_6	geometry
0	2409		Allegheny	2000 S FAYETTE WAY	MCDONALD	02-97992	South Fayette Twp	SOUTHERN BELTWAY MAINT	PA	15057	...	849719	http://cedatareporting.pa.gov/Reportserver/Pag...	55964	PA TURNPIKE COMM	PO BOX 67676		HARRISBURG	PA	17106-7676	None
1	2409		Allegheny	5993 S BRADDOCK AVE	EDGEWOOD	02-55204	Edgewood Boro	GET GO 3147	PA	15218	...	828738	http://cedatareporting.pa.gov/Reportserver/Pag...	125094	GIANT EAGLE INC	101 KAPPA DR		PITTSBURGH	PA	15238	None
2	2443		Lancaster	520 LIME QUARRY RD	GAP	36-09911	Salisbury Twp	LIMEVILLE QUARRY	PA	17527-9739	...	577229	http://cedatareporting.pa.gov/Reportserver/Pag...	62856	NEW ENTERPRISE STONE & LIME CO INC	3912 BRUMBAUGH RD		NEW ENTERPRISE	PA	16664-9137	None
3	2409		Allegheny	3901 CALIFORNIA AVE	PITTSBURGH	02-55599	Pittsburgh City	CALIFORNIA MINI MART	PA	15212-1610	...	842194	http://cedatareporting.pa.gov/Reportserver/Pag...	350369	CALIFORNIA MINI MART LLC	3901 CALIFORNIA AVE		PITTSBURGH	PA	15212-1610	None
4	2429	HANGAR 604	Dauphin	604 AIRPORT DR	MIDDLETOWN	22-64095	Lower Swatara Twp	MIWD FLIGHT HANGER 604	PA	17057-5036	...	572910	http://cedatareporting.pa.gov/Reportserver/Pag...	357377	MIWD FLIGHT LLC	650 W MARKET ST		GRATZ	PA	17030-9701	None

5 rows × 27 columns

In [12]:

new_feature_layer_item = combined_data.spatial.to_featurelayer(new_feature_layer_name, folder = "ArcGIS Python API")

new_feature_layer_item = combined_data.spatial.to_featurelayer(new_feature_layer_name, folder = "ArcGIS Python API")

The operation was attempted on an empty geometry.
The operation was attempted on an empty geometry.

Steps:¶

1. Get Feature Layer Collection:

   • Use FeatureLayerCollection.fromitem() to retrieve the feature layer collection object from the new feature layer item.

2. Overwrite Feature Layer with Shapefile:

   • Call the overwrite() method on the feature layer collection manager to update the feature layer with data from the shapefile.

In [13]:

try:
    # Get the feature layer collection
    featureLayerCollection = FeatureLayerCollection.fromitem(new_feature_layer_item)
    
    # Overwrite the feature layer with the shapefile
    featureLayerCollection.manager.overwrite(shp_file_path)
    print("Feature layer updated successfully.")
except Exception as e:
    print(f"Error creating or updating the new feature layer: {e}")

try: # Get the feature layer collection featureLayerCollection = FeatureLayerCollection.fromitem(new_feature_layer_item) # Overwrite the feature layer with the shapefile featureLayerCollection.manager.overwrite(shp_file_path) print("Feature layer updated successfully.") except Exception as e: print(f"Error creating or updating the new feature layer: {e}")

Error creating or updating the new feature layer: Error while analyzing File Geodatabase 'StorageTankLocations_Active2024_05.shp' Unable to extract 'file geodatabase' Invalid File Geodatabase, missing gdbtable or gdbindexes file.
(Error Code: 406)

In [14]:

os.remove(shp_file_path)

os.remove(shp_file_path)