From d1a4820a357818af4df33125871a28be2cfe3902 Mon Sep 17 00:00:00 2001 From: marcel-dempers Date: Wed, 26 Oct 2022 13:54:42 +1100 Subject: [PATCH] update chapters 1 and 2 --- .../postgres/1-introduction/README.md | 24 +++-- .../postgres/2-configuration/README.md | 95 ++++++++++++++++--- .../2-configuration/{ => config}/pg_hba.conf | 0 .../2-configuration/config/pg_ident.conf | 42 ++++++++ .../2-configuration/config/postgresql.conf | 27 ++++++ .../postgres/3-replication/README.md | 0 6 files changed, 164 insertions(+), 24 deletions(-) rename storage/databases/postgres/2-configuration/{ => config}/pg_hba.conf (100%) create mode 100644 storage/databases/postgres/2-configuration/config/pg_ident.conf create mode 100644 storage/databases/postgres/2-configuration/config/postgresql.conf create mode 100644 storage/databases/postgres/3-replication/README.md diff --git a/storage/databases/postgres/1-introduction/README.md b/storage/databases/postgres/1-introduction/README.md index d00e2e1..2f6e708 100644 --- a/storage/databases/postgres/1-introduction/README.md +++ b/storage/databases/postgres/1-introduction/README.md @@ -20,7 +20,7 @@ cd storage/databases/postgres/1-introduction docker compose up ``` -We can access our database from the adminer webpage on `http://localhost8080`
+We can access our database from the adminer web page on `http://localhost8080`
When running containers, its always important to pull the image by tag
@@ -29,10 +29,10 @@ We will do that in the next step.
## Persisting Data -To persist data to Postgres, we simply mount a docker volume.
+To persist data to PostgreSQL, we simply mount a docker volume.
This is the way to persist container data.
-Postgres stores its data by default under `/var/lib/postgresql/data` -Also take note we are running a specific version of Postges now: +PostgreSQL stores its data by default under `/var/lib/postgresql/data` +Also take note we are running a specific version of PostgreSQL now: ``` docker run -it --rm --name postgres ` @@ -77,8 +77,8 @@ Run it again with the above `docker run` command and list our record with the ab ## Networking -Postgres by default uses port `5432`.
-Since we are running in Docker, we can bind a differnt port if we wish with Docker's `-p` flag.
+PostgreSQL by default uses port `5432`.
+Since we are running in Docker, we can bind a different port if we wish with Docker's `-p` flag.
For example, we can expose port `5000` outside the container : ``` @@ -88,15 +88,15 @@ docker run -it --rm --name postgres ` -p 5000:5432 ` postgres:15.0 ``` -Note that this does not change the port which Postgres runs on.
+Note that this does not change the port which PostgreSQL runs on.
To change that, we need to explore the configuration. ## Configuration PostgreSQL can be configured using environment variables as well as a config file.
-Postgres has a ton of configuration options.
-In the next chapter, we will explore the configuration of Postgres.
+PostgreSQL has a ton of configuration options.
+In the next chapter, we will explore the configuration of PostgreSQL.
## Docker Compose @@ -121,4 +121,8 @@ services: restart: always ports: - 8080:8080 -``` \ No newline at end of file +``` + +That's it for chapter one!
+In [chapter 2](../2-configuration/README.md), we will take a look at Configuration and how to start our PostgreSQL instance with a custom configuration file.
+We will also explore the customization options available.
\ No newline at end of file diff --git a/storage/databases/postgres/2-configuration/README.md b/storage/databases/postgres/2-configuration/README.md index d27380f..0569fa3 100644 --- a/storage/databases/postgres/2-configuration/README.md +++ b/storage/databases/postgres/2-configuration/README.md @@ -51,7 +51,7 @@ If we take a look at our `docker` mount that we defined in our `docker run` comm `-v ${PWD}/pgdata:/var/lib/postgresql/data `
-The `{PWD}/pgdata` folder that we have mounted contains not only data, but some defaut configuration files that we can explore.
+The `{PWD}/pgdata` folder that we have mounted contains not only data, but some default configuration files that we can explore.
Three files are important here: @@ -106,17 +106,84 @@ This is not a feature that we will need in this series, so we will skip this con This configuration file is the main one for PostgreSQL.
As you can see this is a large file with in-depth tuning and customization capability.
+### File Locations + +Let's set our data directory locations as well as config file locations
+Our volume mount path in the container is also short and simple.
+Note that we also split config from data so we have separate paths : + ``` -docker run -d --rm --name postgres-1 ` - --net postgres ` - -e POSTGRES_USER=postgresadmin ` - -e POSTGRES_PASSWORD=admin123 ` - -e POSTGRES_DB=postgresdb ` - -e PGDATA=/var/lib/postgresql/data/pgdata ` - -p 5000:5432 ` - -v ${PWD}/archive:/mnt/server/archive ` - -v ${PWD}/postgres-1/pgdata:/var/lib/postgresql/data/pgdata ` - -v ${PWD}/postgres-1/postgresql.conf:/etc/postgresql/postgresql.conf ` - -v ${PWD}/postgres-1/pg_hba.conf:/var/lib/postgresql/data/pgdata/pg_hba.conf ` - postgres:14.4 -c 'config_file=/etc/postgresql/postgresql.conf' -``` \ No newline at end of file +data_directory = '/data' +hba_file = '/config/pg_hba.conf' +ident_file = '/config/pg_ident.conf' +``` + +### Connection and Authentication + +The shared_buffers parameter determines how much memory is dedicated to the server for caching data. The value should be set to 15% to 25% of the machine's total RAM. For example: if your machine's RAM size is 32 GB, then the recommended value for shared_buffers is 8 GB
+ +We will take a look at `WAL` (Write Ahead Log), Archiving, Primary, and Standby configurations in a future chapter on replication
+ +``` +port = 5432 +listen_addresses = '*' +max_connections = 100 +shared_buffers = 128MB +dynamic_shared_memory_type = posix +max_wal_size = 1GB +min_wal_size = 80MB +log_timezone = 'Etc/UTC' +datestyle = 'iso, mdy' +timezone = 'Etc/UTC' + +#locale settings +lc_messages = 'en_US.utf8' # locale for system error message +lc_monetary = 'en_US.utf8' # locale for monetary formatting +lc_numeric = 'en_US.utf8' # locale for number formatting +lc_time = 'en_US.utf8' # locale for time formatting + +default_text_search_config = 'pg_catalog.english' + +``` + +We can also include other configurations from other locations with the `include_dir` and `include` options.
+We will skip these for the sake of keeping things simple.
+Nested configurations can over complicate a setup and makes it hard to troubleshoot when issues occur.
+ +### Specifying Custom Configuration + +If we run on Linux, we need to ensure that the `postgres` user which has a user ID of `999` by default, should have access to the configuration files.
+ +``` +sudo chown 999:999 config/postgresql.conf +sudo chown 999:999 config/pg_hba.conf +sudo chown 999:999 config/pg_ident.conf +``` + +There is another important gotcha here.
+The `PGDATA` variable tells PostgreSQL where our data directory is.
+Similarly, we've learnt that our configuration file also has `data_directory` which tells PostgreSQL the same.
+ +However, the latter is only read by PostgreSQL after initialization has occurred.
+PostgreSQL's initialization phase sets up directory permissions on the data directory.
+If we leave out `PGDATA`, then we will get errors that the data directory is invalid.
+Hence `PGDATA` is important here.
+ +## Running our PostgreSQL + +Finally, we can run our database with our custom configuration files: + +``` +docker run -it --rm --name postgres ` +-e POSTGRES_USER=postgresadmin ` +-e POSTGRES_PASSWORD=admin123 ` +-e POSTGRES_DB=postgresdb ` +-e PGDATA="/data" ` +-v ${PWD}/pgdata:/data ` +-v ${PWD}/config:/config ` +-p 5000:5432 ` +postgres:15.0 -c 'config_file=/config/postgresql.conf' +``` + +That's it for chapter two!
+In [chapter 3](../3-replication/README.md), we will take a look at Replication and how to replicate our data to another PostgreSQL instance for better availability. \ No newline at end of file diff --git a/storage/databases/postgres/2-configuration/pg_hba.conf b/storage/databases/postgres/2-configuration/config/pg_hba.conf similarity index 100% rename from storage/databases/postgres/2-configuration/pg_hba.conf rename to storage/databases/postgres/2-configuration/config/pg_hba.conf diff --git a/storage/databases/postgres/2-configuration/config/pg_ident.conf b/storage/databases/postgres/2-configuration/config/pg_ident.conf new file mode 100644 index 0000000..a5870e6 --- /dev/null +++ b/storage/databases/postgres/2-configuration/config/pg_ident.conf @@ -0,0 +1,42 @@ +# PostgreSQL User Name Maps +# ========================= +# +# Refer to the PostgreSQL documentation, chapter "Client +# Authentication" for a complete description. A short synopsis +# follows. +# +# This file controls PostgreSQL user name mapping. It maps external +# user names to their corresponding PostgreSQL user names. Records +# are of the form: +# +# MAPNAME SYSTEM-USERNAME PG-USERNAME +# +# (The uppercase quantities must be replaced by actual values.) +# +# MAPNAME is the (otherwise freely chosen) map name that was used in +# pg_hba.conf. SYSTEM-USERNAME is the detected user name of the +# client. PG-USERNAME is the requested PostgreSQL user name. The +# existence of a record specifies that SYSTEM-USERNAME may connect as +# PG-USERNAME. +# +# If SYSTEM-USERNAME starts with a slash (/), it will be treated as a +# regular expression. Optionally this can contain a capture (a +# parenthesized subexpression). The substring matching the capture +# will be substituted for \1 (backslash-one) if present in +# PG-USERNAME. +# +# Multiple maps may be specified in this file and used by pg_hba.conf. +# +# No map names are defined in the default configuration. If all +# system user names and PostgreSQL user names are the same, you don't +# need anything in this file. +# +# This file is read on server startup and when the postmaster receives +# a SIGHUP signal. If you edit the file on a running system, you have +# to SIGHUP the postmaster for the changes to take effect. You can +# use "pg_ctl reload" to do that. + +# Put your actual configuration here +# ---------------------------------- + +# MAPNAME SYSTEM-USERNAME PG-USERNAME diff --git a/storage/databases/postgres/2-configuration/config/postgresql.conf b/storage/databases/postgres/2-configuration/config/postgresql.conf new file mode 100644 index 0000000..6b49304 --- /dev/null +++ b/storage/databases/postgres/2-configuration/config/postgresql.conf @@ -0,0 +1,27 @@ +# ----------------------------- +# PostgreSQL configuration file +# ----------------------------- +# + +data_directory = '/data' +hba_file = '/config/pg_hba.conf' +ident_file = '/config/pg_ident.conf' + +port = 5432 +listen_addresses = '*' +max_connections = 100 +shared_buffers = 128MB +dynamic_shared_memory_type = posix +max_wal_size = 1GB +min_wal_size = 80MB +log_timezone = 'Etc/UTC' +datestyle = 'iso, mdy' +timezone = 'Etc/UTC' + +#locale settings +lc_messages = 'en_US.utf8' # locale for system error message +lc_monetary = 'en_US.utf8' # locale for monetary formatting +lc_numeric = 'en_US.utf8' # locale for number formatting +lc_time = 'en_US.utf8' # locale for time formatting + +default_text_search_config = 'pg_catalog.english' \ No newline at end of file diff --git a/storage/databases/postgres/3-replication/README.md b/storage/databases/postgres/3-replication/README.md new file mode 100644 index 0000000..e69de29