From 2f778931f34339a92843f859b271782cae8c9924 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Wed, 1 Sep 2021 22:11:28 +1000 Subject: [PATCH] json --- python/introduction/part-2.files/README.md | 8 +- python/introduction/part-3.json/README.md | 301 +++++++++++++++++++++ python/introduction/part-3.json/dockerfile | 8 + python/introduction/part-3.json/src/app.py | 50 ++++ 4 files changed, 363 insertions(+), 4 deletions(-) create mode 100644 python/introduction/part-3.json/README.md create mode 100644 python/introduction/part-3.json/dockerfile create mode 100644 python/introduction/part-3.json/src/app.py diff --git a/python/introduction/part-2.files/README.md b/python/introduction/part-2.files/README.md index d803afc..9621af9 100644 --- a/python/introduction/part-2.files/README.md +++ b/python/introduction/part-2.files/README.md @@ -6,7 +6,7 @@ programming for a number of reasons:
* Applications may need to read configuration files * In Data science, data is often sourced from files (`CSV`, `XML`, `JSON`, etc) * Data is often analysed in Python when its written in different stages of analysis -* DevOps engineers often store state of infrastructure or data as files for automation purposes. +* DevOps engineers often stores the state of infrastructure or data as files for automation purposes. Files are not the endgame for storage.
Remember there are things like Caches and Databases.
@@ -66,7 +66,7 @@ def getCustomer(customerID): ## Opening Files Python provides an `open` function to open files.
-`open()` takes a file path\name and acccess mode +`open()` takes a file path\name and access mode ``` "r" - Read - Default value. Opens a file for reading, error if the file does not exist @@ -92,7 +92,7 @@ FileNotFoundError: [Errno 2] No such file or directory: 'customers.log' ``` Let's use what we learned (`if` statements), to check if the file exists! -We'll need a built in library for handing files +We'll need a built in library for handling files ``` import os.path @@ -128,7 +128,7 @@ for customer in f: f.close() ``` -Now we know the file does not exist, lets create it! +Now we know the file does not exist, let's create it! ``` customers = getCustomers() diff --git a/python/introduction/part-3.json/README.md b/python/introduction/part-3.json/README.md new file mode 100644 index 0000000..b099e66 --- /dev/null +++ b/python/introduction/part-3.json/README.md @@ -0,0 +1,301 @@ +# Introduction to Python: JSON + +JSON is a very important format for storing data in Software engineering.
+JSON is popular for the following scenarios : + +* Popular standard for data structures in general. +* Applications may talk to each other by passing JSON data back and forth (HTTP Web APIs). +* Applications often store configuration in JSON format as a configuration file. +* Data may be stored in databases or caches in JSON format. +* DevOps engineers may store infrastructure configuration in JSON format. + +## Python Dev Environment + +The same as Part 1, we start with a [dockerfile](./dockerfile) where we declare our version of `python`. + +``` +FROM python:3.9.6-alpine3.13 as dev + +WORKDIR /work +``` + +Let's build and start our container: + +``` +cd python\introduction\part-3.json + +docker build --target dev . -t python +docker run -it -v ${PWD}:/work python sh + +/work # python --version +Python 3.9.6 + +``` + +## Our application + +Firstly we have a class to define what a customer looks like: +``` +class Customer: + def __init__(self, c="",f="",l=""): + self.customerID = c + self.firstName = f + self.lastName = l + def fullName(self): + return self.firstName + " " + self.lastName +``` + +Then we need a function which returns our customers: +``` +def getCustomers(): + customers = { + "a": Customer("a","James", "Baker"), + "b": Customer("b", "Jonathan", "D"), + "c": Customer("c", "Aleem", "Janmohamed"), + "d": Customer("d", "Ivo", "Galic"), + "e": Customer("e", "Joel", "Griffiths"), + "f": Customer("f", "Michael", "Spinks"), + "g": Customer("g", "Victor", "Savkov"), + "h" : Customer("h", "Marcel", "Dempers") + } + return customers +``` + +Here is a function to return a specific customer: +``` +def getCustomer(customerID): + customer = getCustomers() + return customer[customerID] +``` + +Test our functions: + +``` +customers = getCustomers() +customer = customers["h"] + +print(customers) +print(customer.fullName) +``` + +## Files + +In [part-2](../part-2.files/README.md) we learnt about reading and writing to files.
+We changed our functions to get customers from file and update customers by writing the data back to file.
+ +Let's do that quick: + +## Get Customers + +``` +import os.path +import csv + +def getCustomers(): + if os.path.isfile("customers.log"): + with open('customers.log', newline='') as customerFile: + reader = csv.DictReader(customerFile) + l = list(reader) + customers = {c["customerID"]: c for c in l} + return customers + else: + return {} +``` + +## Update Customers + +Let's create a function to update our customers: + +``` +def updateCustomers(customers): + fields = ['customerID', 'firstName', 'lastName'] + with open('customers.log', 'w', newline='') as customerFile: + writer = csv.writer(customerFile) + writer.writerow(fields) + for customerID in customers: + customer = customers[customerID] + writer.writerow([customer.customerID, customer.firstName, customer.lastName]) +``` + +## Test + +Let's test our two functions to ensure they work + +``` +customers = { + "a": Customer("a","James", "Baker"), + "b": Customer("b", "Jonathan", "D"), + "c": Customer("c", "Aleem", "Janmohamed"), + "d": Customer("d", "Ivo", "Galic"), + "e": Customer("e", "Joel", "Griffiths"), + "f": Customer("f", "Michael", "Spinks"), + "g": Customer("g", "Victor", "Savkov"), + "h" : Customer("h", "Marcel", "Dempers") +} + +#save it +updateCustomers(customers) + +#see the changes +customers = getCustomers() +for customer in customers: + print(customers[customer]) + +``` + +## JSON + +JSON is a very simple data structure for building objects. +you can define any type of object and data using JSON.
+ +In JSON, objects are defined by open and close brackets, like `{}`
+Arrays or lists are defined as square brackets, like `[]`
+ +We can therefore define a customer like : +`Notice keys and values have quotes, and separated by colon` + +``` +{ + "customerID" : "a", + "firstName: "Bob", + "lastName": "Smith" +} + +``` +We can define an array of customers in JSON by declaring our array with square brackets and having customer objects inside, separated by commas: + +``` +# Example Array: +[{customer-1},{customer-2},{customer-3},{customer-4} ] +``` + +## JSON Library + +Python provides a library for dealing with JSON.
+Let's take our customer dictionary, and convert it to json structure and +finally write it to file using our update function. + +``` +import json +``` + +Now first things first, we need to convert our customers dictionary to a JSON structured string so we can store it to file.
+ +The `json` library allows us to convert dictionaries to files using +the `dumps` function: + +``` +jsonData = json.dumps(customers) +print(jsonData) +``` + +But if we run this we get an error: +`TypeError: Object of type Customer is not JSON serializable` + +That is because the library can only deal with full dictionaries. +Taking a closer look at our dictionary, our customer is a class. + +``` +# our key is a string +# our value is a class +"h" : Customer("h", "Marcel", "Dempers") +``` + +`json.dumps()` only allows full dictionaries so we need our customer object to be in dictionary form like so: + +``` +"h": { "customerID": "h", "firstName" : "Marcel", "lastName" : "Dempers"} +``` + +To fix our dictionary, we can iterate it and build a new one + +``` +# start with an empty one +customerDict = {} + +#populate our new dictionary +for id in customers: + customerDict[id] = customers[id].__dict__ + +#print it +print(customerDict) +``` + +Now we can convert our customer dictionary to json + +``` +customerJSON = json.dumps(customerDict) +print(customerJSON) +``` + +Now we've converted our data into JSON, we can view this in a new tab window and change the language mode to JSON to visualise the data in our new format. + +Now essentially our application will use dictionaries when working with the data +but store and transport it as JSON.
+ +Therefore our update function should take a dictionary and convert it to JSON +and store it. + +``` +def updateCustomers(customer): + with open('customers.json', 'w', newline='') as customerFile: + customerJSON = json.dumps(customer) + customerFile.write(customerJSON) +``` + +We also need to change our `getCustomers` function to read the JSON file +and convert that data correctly back to a dictionary for processing. + +``` +def getCustomers(): + if os.path.isfile("customers.json"): + with open('customers.json', newline='') as customerFile: + data = customerFile.read() + customers = json.loads(data) + return customers + else: + return {} +``` + +Finally let's test our functions : + +``` +customers = getCustomers() +print(customers) +``` + +It's also very easy to add a customer to our JSON data using our class and using pythons internal `__dict__` property to convert the object to a dictionary and add it to our customers dictionary + +``` +customers["i"] = Customer("i", "Bob", "Smith").__dict__ +updateCustomers(customers) +``` + +## Docker + +Let's build our container image and run it while mounting our customer file + +Our final `dockerfile` +``` +FROM python:3.9.6-alpine3.13 as dev + +WORKDIR /work + +FROM dev as runtime +COPY ./src/ /app + +ENTRYPOINT [ "python", "/app/app.py" ] + +``` + +Build and run our container. +Notice the `customers.json` file gets created if it does not exist. + +``` +cd python\introduction\part-3.json + +docker build . -t customer-app + +docker run -v ${PWD}:/work -w /work customer-app + +``` \ No newline at end of file diff --git a/python/introduction/part-3.json/dockerfile b/python/introduction/part-3.json/dockerfile new file mode 100644 index 0000000..7666e03 --- /dev/null +++ b/python/introduction/part-3.json/dockerfile @@ -0,0 +1,8 @@ +FROM python:3.9.6-alpine3.13 as dev + +WORKDIR /work + +FROM dev as runtime +COPY ./src/ /app + +ENTRYPOINT [ "python", "/app/app.py" ] \ No newline at end of file diff --git a/python/introduction/part-3.json/src/app.py b/python/introduction/part-3.json/src/app.py new file mode 100644 index 0000000..c070912 --- /dev/null +++ b/python/introduction/part-3.json/src/app.py @@ -0,0 +1,50 @@ +import os.path +import csv +import json + +class Customer: + def __init__(self, c="",f="",l=""): + self.customerID = c + self.firstName = f + self.lastName = l + def fullName(self): + return self.firstName + " " + self.lastName + +def getCustomers(): + if os.path.isfile("customers.json"): + with open('customers.json', newline='') as customerFile: + data = customerFile.read() + customers = json.loads(data) + return customers + else: + return {} + +def updateCustomers(customer): + with open('customers.json', 'w', newline='') as customerFile: + customerJSON = json.dumps(customer) + customerFile.write(customerJSON) + +customers = { + "a": Customer("a","James", "Baker"), + "b": Customer("b", "Jonathan", "D"), + "c": Customer("c", "Aleem", "Janmohamed"), + "d": Customer("d", "Ivo", "Galic"), + "e": Customer("e", "Joel", "Griffiths"), + "f": Customer("f", "Michael", "Spinks"), + "g": Customer("g", "Victor", "Savkov"), + "h" : Customer("h", "Marcel", "Dempers") +} + + +customerDict = {} +for id in customers: + customerDict[id] = customers[id].__dict__ + +updateCustomers(customerDict) + +customers = getCustomers() + +customers["i"] = Customer("i", "Marcel", "Dempers").__dict__ +updateCustomers(customers) + +print(customers)