Extract data from a map service using Python

Summary

In some cases, users enable the Feature Access function on map services to allow easy download of data. However, data can be downloaded from a map service in the form of JSON and the JSON code can be converted to a shapefile or Feature Class. This article describes how to do so with a publicly shared map service using ArcPy and other built-in Python libraries.

Procedure

The following instructions demonstrate how to query a map service for features, write the JSON response to a file, and convert the JSON file to a shapefile using the arcpy.JSONToFeatures_conversion() function. It is possible to use the script as-is, or format them into functions that take a URL argument.

Adjust the query parameters as needed (most, but not all, of the possible parameters are included). Output files are stored in the directory containing the Python script.

1. Import the necessary libraries.

# For Python 2
import urllib2, urllib, os, arcpy

# For Python 3
import urllib.parse, urllib.request, os, arcpy, json

2. If a token is required to access a secured service, use the snippet below. If not, skip to step 3.

username = "adminaccount"
password = "adminpassword"

tokenURL = 'https://machine.domain.com/portal/sharing/rest/generateToken/'
params = {'f': 'pjson', 'username': username, 'password': password, 'referer': 'https://machine.domain.com'}

#For Python 2
req = urllib2.Request(tokenURL, urllib.urlencode(params))
response = urllib2.urlopen(req)

#For Python 3
data = urllib.parse.urlencode(query=params).encode('utf-8')
req = urllib.request.Request(tokenURL, data)
response = urllib.request.urlopen(req)

data = json.loads(response.read())
token = data['token']

3. Specify the desired URL.

url = "https://sampleserver6.arcgisonline.com/arcgis/rest/services/Census/MapServer/3/query?"

4. Query the parameters.

#For Python 2
params = urllib.urlencode({'where': '1=1',
		   'geometryType': 'esriGeometryEnvelope',
		   'spatialRel': 'esriSpatialRelIntersects',
		   'relationParam': '',
		   'outFields': '*',
		   'returnGeometry': 'true',
		   'geometryPrecision':'',
		   'outSR': '',
		   'returnIdsOnly': 'false',
		   'returnCountOnly': 'false',
		   'orderByFields': '',
		   'groupByFieldsForStatistics': '',
		   'returnZ': 'false',
		   'returnM': 'false',
		   'returnDistinctValues': 'false',
		   'f': 'pjson'
		   })

# For Python 3
params = {'where': '1=1',
		   'geometryType': 'esriGeometryEnvelope',
		   'spatialRel': 'esriSpatialRelIntersects',
		   'relationParam': '',
		   'outFields': '*',
		   'returnGeometry': 'true',
		   'geometryPrecision':'',
		   'outSR': '',
		   'returnIdsOnly': 'false',
		   'returnCountOnly': 'false',
		   'orderByFields': '',
		   'groupByFieldsForStatistics': '',
		   'returnZ': 'false',
		   'returnM': 'false',
		   'returnDistinctValues': 'false',
		   'f': 'pjson',
                   'token': token
		   }

encode_params = urllib.parse.urlencode(params).encode("utf-8")

5. Create a request and read it using urllib.

# For Python 2
request = urllib2.Request(url, params)
response = urllib2.urlopen(request)
json = response.read()

# For Python 3
response = urllib.request.urlopen(url, encode_params)
json = response.read()

6. Write the JSON response to text file.

with open("mapservice.json", "wb") as ms_json:
    ms_json.write(json)

7. Convert JSON to shapefile using the JSONToFeatures function.

ws = os.getcwd() + os.sep
arcpy.JSONToFeatures_conversion("mapservice.json", ws + "mapservice.shp")

import urllib2, urllib, os, arcpy

url = "https://sampleserver6.arcgisonline.com/arcgis/rest/services/Census/MapServer/3/query?"

params = urllib.urlencode({'where': '1=1',
		   'geometryType': 'esriGeometryEnvelope',
		   'spatialRel': 'esriSpatialRelIntersects',
		   'relationParam': '',
		   'outFields': '*',
		   'returnGeometry': 'true',
		   'geometryPrecision':'',
		   'outSR': '',
		   'returnIdsOnly': 'false',
		   'returnCountOnly': 'false',
		   'orderByFields': '',
		   'groupByFieldsForStatistics': '',
		   'returnZ': 'false',
		   'returnM': 'false',
		   'returnDistinctValues': 'false',
		   'f': 'pjson'
		   })

request = urllib2.Request(url, params)
response = urllib2.urlopen(request)
json = response.read()

with open("mapservice.json", "wb") as ms_json:
    ms_json.write(json)

ws = os.getcwd() + os.sep
arcpy.JSONToFeatures_conversion("mapservice.json", ws + "mapservice.shp")

import urllib.parse, urllib.request, os, arcpy, json

arcpy.env.overwriteOutput = True

# Specify built-in Portal for ArcGIS account credentials
username = "portaluser"
password = "portalpassword"

tokenURL = 'https://server.domain.com/portal/sharing/rest/generateToken/'
params = {'f': 'pjson', 'username': username, 'password': password, 'referer': 'https://server.domain.com'}

data = urllib.parse.urlencode(query=params).encode('utf-8')
req = urllib.request.Request(tokenURL, data)
response = urllib.request.urlopen(req)

data = json.loads(response.read())
token = data['token']

# Specify REST URL for service JSON to be returned
url = "https://server.domain.com/server/rest/services/servicename/MapServer/0/query?"

params = {'where': '1=1',
          'geometryType': 'esriGeometryEnvelope',
          'spatialRel': 'esriSpatialRelIntersects',
          'relationParam': '',
          'outFields': '*',
          'returnGeometry': 'true',
          'geometryPrecision': '',
          'outSR': '',
          'returnIdsOnly': 'false',
          'returnCountOnly': 'false',
          'orderByFields': '',
          'groupByFieldsForStatistics': '',
          'returnZ': 'false',
          'returnM': 'false',
          'returnDistinctValues': 'false',
          'f': 'pjson',
          'token': token
          }

encode_params = urllib.parse.urlencode(params).encode("utf-8")

response = urllib.request.urlopen(url, encode_params)
json = response.read()

with open("mapservice.json", "wb") as ms_json:
    ms_json.write(json)

ws = os.getcwd() + os.sep
arcpy.JSONToFeatures_conversion("mapservice.json", ws + "mapservice.shp", )