CSRF problems - expected token differs from given token

Trying out Kratos and Oathkeeper. We keep getting the following CSRF error. We are running everything in a single docker network, from the same domain with no sub-domains. Oathkeeper is proxying requests from an Angular SPA to Kratos. We get this when we try to login or register. We walked through the CSRF Common Pitfalls documentation but could not find anything relevant to our setup. Any advice?

kratos_1 | time="2020-05-22T20:39:18Z" level=warning msg="A request failed due to a missing or invalid csrf_token value" expected_token="ITJPquqzkfPAgCjSGPALrqJdOCSgVn54rJcOOUpwO2jqoFnf1TmrKJwfLV0lDnSRvlPO/zy3/gbQXznqIRBK6g==" received_token="d71HyUZ9mcqlGS5J2CoQorCgI1GRiV dMrefviuY6T 8L1G8efejEfmGK8bl1G drK7Vig1o3 NOf6htQPiYvQ==" received_token_form="d71HyUZ9mcqlGS5J2CoQorCgI1GRiV dMrefviuY6T 8L1G8efejEfmGK8bl1G drK7Vig1o3 NOf6htQPiYvQ=="

Thank you for the post, please include more information and be very, very specific. There are many set ups possible and I can’t help if I have to guess.

There is also a very large section on CSRF in the documentation, I guess it will be quite helpful: https://www.ory.sh/kratos/docs/debug/csrf

We are using Docker, Oathkeeper, Kratos, and Angular as a SPA.
kratos.json

    {

        "selfservice": {

            "strategies": {

                "password": {

                    "enabled": true

                }

            },

            "settings": {

                "privileged_session_max_age": "1m",

                "after": {

                    "profile": {

                        "hooks": [

                            {

                                "hook": "verify"

                            }

                        ]

                    }

                }

            },

            "verify": {

                "return_to": "https://myapp.com/"

            },

            "logout": {

                "redirect_to": "https://myapp.com/"

            },

            "login": {

                "request_lifespan": "10m"

            },

            "registration": {

                "request_lifespan": "10m",

                "after": {

                    "password": {

                        "hooks": [

                            {

                                "hook": "session"

                            },

                            {

                                "hook": "verify"

                            }

                        ]

                    }

                }

            }

        },

        "log": {

            "level": "debug"

        },

        "secrets": {

            "session": [

                "PLEASE-CHANGE-ME-I-AM-VERY-INSECURE"

            ]

        },

        "urls": {

            "login_ui": "https://myapp.com/auth/login",

            "registration_ui": "https://myapp.com/auth/registration",

            "error_ui": "https://myapp.com/error",

            "settings_ui": "https://myapp.com/settings",

            "verify_ui": "https://myapp.com/verify",

            "mfa_ui": "https://myapp.com/",

            "self": {

                "public": "https://myapp.com/.ory/kratos/public/",

                "admin": "http://kratos:4434/"

            },

            "default_return_to": "https://myapp.com/",

            "whitelisted_return_to_urls": [

                "https://myapp.com"

            ]

        },

        "hashers": {

            "argon2": {

                "parallelism": 1,

                "memory": 131072,

                "iterations": 2,

                "salt_length": 16,

                "key_length": 16

            }

        },

        "identity": {

            "traits": {

                "default_schema_url": "file:///etc/config/kratos/identity.traits.schema.json"

            }

        },

        "courier": {

            "smtp": {

                "connection_uri": "smtps://test:[email protected]:1025/?skip_ssl_verify=true"

            }

        }

    }

oathkeeper.config

    serve:
      proxy:
        port: 4455 # run the proxy at port 4455
        tls:
          key:
            path: /certs/myapp_root.key
          cert:
            path: /certs/myapp_root.crt
        cors:
            enabled: true
            allowed_origins:
              - "*"
            allowed_methods:
            - PATCH
            - PUT
            - GET
            - POST
            - DELETE
            allowed_headers:
                - Authorization
                - Content-Type
            exposed_headers:
                - Content-Type
            allow_credentials: true
            max_age: 0
            debug: true
      api:
        port: 4456 # run the api at port 4456
    access_rules:
      repositories:
        - file:///config/rules.json
    errors:
      fallback:
        - json
      handlers:
        json:
          enabled: true
          config:
            verbose: true
        redirect:
          enabled: true
          config:
            to: https://www.ory.sh/docs
    mutators:
      header:
        enabled: true
        config:
          headers:
            X-User: "{{ print .Subject }}"
      noop:
        enabled: true
    authorizers:
      allow:
        enabled: true
      deny:
        enabled: true
    authenticators:
      noop:
        enabled: true
      anonymous:
        enabled: true
        config:
          subject: guest

oathkeeper rules.json

[
	{
		"id": "kratos-public",
		"version": "v0.36.0-beta.4",
		"upstream": {
			"url": "http://kratos:4433",
			"preserve_host": true,
			"strip_path": "/.ory/kratos/public"
		},
		"match": {
			"url": "https://myapp.com/.ory/kratos/public/<.*>",
			"methods": [
				"GET",
				"POST",
				"PUT",
				"DELETE",
				"PATCH",
				"OPTIONS"
			]
		},
		"authenticators": [
			{
				"handler": "noop"
			}
		],
		"authorizer": {
			"handler": "allow"
		},
		"mutators": [
			{
				"handler": "noop"
			}
		]
	},
	{
		"id": "kratos-public",
		"version": "v0.36.0-beta.4",
		"upstream": {
			"url": "http://kratos:4434"
		},
		"match": {
			"url": "https://myapp.com/self-service/<.*>",
			"methods": [
				"GET",
				"POST",
				"PUT",
				"DELETE",
				"PATCH",
				"OPTIONS"
			]
		},
		"authenticators": [
			{
				"handler": "noop"
			}
		],
		"authorizer": {
			"handler": "allow"
		},
		"mutators": [
			{
				"handler": "noop"
			}
		]
	},
	{
		"id": "allow-frontend",
		"version": "v0.36.0-beta.4",
		"upstream": {
			"url": "http://frontend:4200"
		},
		"match": {
			"url": "https://myapp.com/<(?!(self-service|\\.ory)).*>",
			"methods": [
				"GET"
			]
		},
		"authenticators": [
			{
				"handler": "noop"
			}
		],
		"authorizer": {
			"handler": "allow"
		},
		"mutators": [
			{
				"handler": "noop"
			}
		]
	}
]

Here is what’s happening:

  1. In Chrome, navigate to
    myapp.com/auth/login.
  2. The browser then redirects to the Kratos public API:
    https://myapp.com/.ory/kratos/public/self-service/browser/flows/login.
  3. Kratos redirects the browser back to the application with a requestId:
https://myapp.com/auth/login?request=23c2ae38-30bd-4927-bacc-ef1b514a01f9

Looking at the dev tools, we can see that a cookie is set by Kratos:

csrf_token: tfRs43fajJ04EqumSKz1bfOW02CmI4ktxxSflpWMKPU=
  1. Kratos API returns a JSON response with a csrf_token:
zqUjPw6C1JVt5u+QWM3PDmAEOZ41Dx1xkuPjwakB7nd7UU/ceVhYCFX0RDYQYTpjk5Lq/pMslFxV93xXPI3Ggg
  1. Using this token in the form data, attempt to send a login request:
    https://myapp.com/.ory/kratos/public/self-service/browser/flows/login/strategies/password?request=23c2ae38-30bd-4927-bacc-ef1b514a01f9.

  2. Note that form data is being sent with Content-Type application/x-www-form-urlencoded. Also note that the csrf_token sent with the HTML form is different than the one in the cookie set by Kratos during the initial redirect (step 3)

  3. The login request fails with the message:

"error": {
    "error": {
      "code": 400,
      "status": "Bad Request",
      "reason": "CSRF token is missing or invalid.",
      "message": "The request was malformed or contained invalid parameters"
    }
  }
  1. Looking at the logs, it appears that Kratos is expecting a csrf_token that is different than either the one it set initially (step 3) or the one sent with the HTML form (step 4)
oathkeeper    | [cors] 2020/05/26 14:31:55 Handler: Actual request
oathkeeper    | [cors] 2020/05/26 14:31:55   Actual request no headers added: missing origin
kratos_1          | time="2020-05-26T14:31:55Z" level=info msg="started handling request" method=POST name="public#https://myapp.com/.ory/kratos/public/" remote="172.25.0.4:50326" request="/self-service/browser/flows/login/strategies/password?request=20b32d76-ed4d-49d2-993b-1a0016b2855a"
kratos_1          | time="2020-05-26T14:31:55Z" level=warning msg="A request failed due to a missing or invalid csrf_token value" expected_token="jNeIRfLLQeKwqOawWnIPJzHqNfIZqinagChJPRv2Pek5I+SmhRHNf4i6TRYS3vpKwnzmkr+JoPdHPNarjnoVHA==" received_token="zqUjPw6C1JVt5u QWM3PDmAEOZ41Dx1xkuPjwakB7nd7UU/ceVhYCFX0RDYQYTpjk5Lq/pMslFxV93xXPI3Ggg==" received_token_form="zqUjPw6C1JVt5u QWM3PDmAEOZ41Dx1xkuPjwakB7nd7UU/ceVhYCFX0RDYQYTpjk5Lq/pMslFxV93xXPI3Ggg=="
kratos_1          | time="2020-05-26T14:31:55Z" level=error msg="An error occurred while handling a request" code=400 debug= details="map[]" error="The request was malformed or contained invalid parameters" reason="CSRF token is missing or invalid." request-id= status=400 writer=JSON
kratos_1          | time="2020-05-26T14:31:55Z" level=info msg="completed handling request" method=POST name="public#https://myapp.com/.ory/kratos/public/" remote="172.25.0.4:50326" request="/self-service/browser/flows/login/strategies/password?request=20b32d76-ed4d-49d2-993b-1a0016b2855a" status=400 text_status="Bad Request" took="870.1µs"
  1. As you can see
expected_token="jNeIRfLLQeKwqOawWnIPJzHqNfIZqinagChJPRv2Pek5I+SmhRHNf4i6TRYS3vpKwnzmkr+JoPdHPNarjnoVHA=="

is not the same as either the cookie token or the token in the HTML form:
Cookie in browser, sent in header:

tfRs43fajJ04EqumSKz1bfOW02CmI4ktxxSflpWMKPU=

CSRF_Token sent back with form data:

zqUjPw6C1JVt5u+QWM3PDmAEOZ41Dx1xkuPjwakB7nd7UU/ceVhYCFX0RDYQYTpjk5Lq/pMslFxV93xXPI3Ggg

Have you tried inspecting the cookies as documented here: https://www.ory.sh/kratos/docs/debug/csrf/#common-pitfalls ?

Another issue could be that you have a proxy in front (e.g. a load balancer) which strips the cookies.

Yes. As stated, the cookie visible in the browsers dev tools is one value, the one sent back in the response from Kratos is another one, and the logs yet expect a totally different one. There are three distinct csrf tokens being set/sent/expected and we are trying to figure out where the disconnect is.

Ok so what would be really helpful is if you could do the request that fails for you and show me, for each URL call that is either an AJAX call or a regular browser get / redirect request:

Make sure to do the full request from start to finish (e.g. the initial call to the login init endpoint).

Please also include your full Docker Compose config as well as your full ORY Kratos config. Please also upload the file that does the flow of your UI.

Note 1: We modified /etc/hosts to have both myapp.com and localhost point to 127.0.0.1

Note 2: Chrome cookies tab does not show the cookie for some reason, but it is there - you can see it in the request header. It is visible in the Firefox dev tools tab so maybe just a bug in Chrome.

Network Requests and Cookies Screen Shots:
As a new user, I can not upload more than one image, so here is a link to all of them on Imgur: https://imgur.com/a/ZANOoFz.

CSRF Error Logs:

myapp.com         | time="2020-05-29T13:27:40Z" level=info msg="started handling request" method=POST remote="172.29.0.1:34770" request="/.ory/kratos/public/self-service/browser/flows/login/strategies/password?request=9b47ee46-2373-4600-8f5d-ecb2ed0fb511"
kratos            | time="2020-05-29T13:27:40Z" level=info msg="started handling request" method=POST name="public#https://myapp.com/.ory/kratos/public/" remote="172.29.0.5:38576" request="/self-service/browser/flows/login/strategies/password?request=9b47ee46-2373-4600-8f5d-ecb2ed0fb511"
kratos            | time="2020-05-29T13:27:40Z" level=warning msg="A request failed due to a missing or invalid csrf_token value" expected_token="hEgDt2IQv49bdM13rEZiJYbAonMjjvDq0LWlvWTanjMPJe1zxVg7OA/p2NtMZqJOkeJBWkgwDU87UHwe5u0VMQ==" received_token="8p/iXKl/2Riiu9U33FD93TS0iraTFNEoW /68PJKQah58gyYDjddr/YmwJs8cD22I5Zpn/iqLI2wCiNTcH3Kqg==" received_token_form="8p/iXKl/2Riiu9U33FD93TS0iraTFNEoW /68PJKQah58gyYDjddr/YmwJs8cD22I5Zpn/iqLI2wCiNTcH3Kqg=="
kratos            | time="2020-05-29T13:27:40Z" level=error msg="An error occurred while handling a request" code=400 debug= details="map[]" error="The request was malformed or contained invalid parameters" reason="CSRF token is missing or invalid." request-id= status=400 writer=JSON
kratos            | time="2020-05-29T13:27:40Z" level=info msg="completed handling request" method=POST name="public#https://myapp.com/.ory/kratos/public/" remote="172.29.0.5:38576" request="/self-service/browser/flows/login/strategies/password?request=9b47ee46-2373-4600-8f5d-ecb2ed0fb511" status=400 text_status="Bad Request" took="556.6µs"
myapp.com         | time="2020-05-29T13:27:40Z" level=warning msg="Access request granted" granted=true http_host=myapp.com http_method=POST http_url="http://kratos:4433/self-service/browser/flows/login/strategies/password?request=9b47ee46-2373-4600-8f5d-ecb2ed0fb511" http_user_agent="Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.61 Safari/537.36" subject=
myapp.com         | time="2020-05-29T13:27:40Z" level=info msg="completed handling request" measure#oathkeeper-proxy.latency=2443500 method=POST remote="172.29.0.1:34770" request="/.ory/kratos/public/self-service/browser/flows/login/strategies/password?request=9b47ee46-2373-4600-8f5d-ecb2ed0fb511" status=400 text_status="Bad Request" took=2.4435ms

Kratos.json

{
    "selfservice": {
    "strategies": {
        "password": {
        "enabled": true
        }
    },
    "settings": {
        "privileged_session_max_age": "1m"
    },
    "verify": {
        "return_to": "https://myapp.com"
    },
    "logout": {
        "redirect_to": "https://myapp.com"
    },
    "login": {
        "request_lifespan": "10m"
    },
    "registration": {
        "request_lifespan": "10m"
    }
    },
    "log": {
    "level": "debug"
    },
    "secrets": {
    "session": [
        "PLEASE-CHANGE-ME-VERY-INSECURE"
    ]
    },
    "urls": {
    "login_ui": "https://myapp.com/auth/login",
    "registration_ui": "https://myapp.com/auth/registration",
    "error_ui": "https://myapp.com/error",
    "settings_ui": "https://myapp.com/settings",
    "verify_ui": "https://myapp.com/verify",
    "mfa_ui": "https://myapp.com/mfa",
    "self": {
        "public": "https://myapp.com/.ory/kratos/public/",
        "admin": "http://kratos:4434"
    },
    "default_return_to": "https://myapp.com",
    "whitelisted_return_to_urls": [
        "https://myapp.com"
    ]
    },
    "hashers": {
    "argon2": {
        "parallelism": 1,
        "memory": 131072,
        "iterations": 2,
        "salt_length": 16,
        "key_length": 16
    }
    },
    "identity": {
    "traits": {
        "default_schema_url": "file:///etc/config/kratos/identity.traits.schema.json"
    }
    },
    "courier": {
    "smtp": {
        "connection_uri": "smtps://test:[email protected]:1025/?skip_ssl_verify=true"
    }
    }
}

Oathkeeper Rules:

[
    {
    "id": "kratos-public",
    "version": "v0.38.2-beta.1",
    "upstream": {
        "preserve_host": true,
        "url": "http://kratos:4433",
        "strip_path": "/.ory/kratos/public"
    },
    "match": {
        "url": "https://myapp.com/.ory/kratos/public/<.*>",
        "methods": [
        "GET",
        "POST",
        "PUT",
        "DELETE",
        "PATCH"
        ]
    },
    "authenticators": [
        {
        "handler": "noop"
        }
    ],
    "authorizer": {
        "handler": "allow"
    },
    "mutators": [
        {
        "handler": "noop"
        }
    ]
    },
    
    {
    "id": "allow-frontend",
    "version": "v0.38.2-beta.1",
    "upstream": {
        "preserve_host": true,
        "url": "http://frontend"
    },
    "match": {
        "url": "https://myapp.com/<(?!\\.ory).*>",
        "methods": [
        "GET"
        ]
    },
    "authenticators": [
        {
        "handler": "noop"
        }
    ],
    "authorizer": {
        "handler": "allow"
    },
    "mutators": [
        {
        "handler": "noop"
        }
    ]
    }
]

Oathkeeper Config:

serve:
  proxy:
    port: 4455
    tls:
      key:
        path: /certs/myapp.com-key.pem
      cert:
        path: /certs/myapp.com.pem
  api:
    port: 4456
          
access_rules:
  repositories:
    - file:///config/rules.json
errors:
  fallback:
    - json
  handlers:
    json:
      enabled: true
      config:
        verbose: true
    redirect:
      enabled: true
      config:
        to: https://www.ory.sh/docs
mutators:
  noop:
    enabled: true
authorizers:
  allow:
    enabled: true
authenticators:
  noop:
    enabled: true

Docker Compose:

version: '3.7'
services:
  kratos:
    image: oryd/kratos:latest-sqlite
    container_name: kratos
    ports:
      - 4433:4433 # public
      - 4434:4434 # admin
    environment:
      - DSN=sqlite:///var/lib/sqlite/db.sqlite?_fk=true&mode=rwc
    volumes:
      - kratos-sqlite:/var/lib/sqlite
      - ./configs/kratos/:/etc/config/kratos/
    command:
      serve -c /etc/config/kratos/.kratos.json
  kratos-migrate:
    image: oryd/kratos:latest-sqlite
    container_name: kratos_migrate
    volumes:
      - kratos-sqlite:/var/lib/sqlite
      - ./configs/kratos/:/etc/config/kratos/
    environment:
      - DSN=sqlite:///var/lib/sqlite/db.sqlite?_fk=true&mode=rwc
    command:
      -c /etc/config/kratos/.kratos.json migrate sql -e --yes
    restart: on-failure
 
  oathkeeper:
    container_name: myapp.com
    image: oryd/oathkeeper:v0.38.2-beta.1
    ports:
      - 443:4455
      - 4456:4456
    command: serve --config /config/config.yaml
    restart: on-failure
    volumes:
      - ./configs/oathkeeper/:/config/
      - ./configs/certs/:/certs/
  frontend:
    container_name: frontend
    image: frontend
    build:
      context: ./frontend
      dockerfile: Dockerfile
    restart: on-failure
    # volumes:
    #   - ./frontend/src/:/app/src
volumes:
  kratos-sqlite:

Angular:

kratosAuth(url: string, body: any): Observable<any>{
    return this.http.post<any>(
        url,
        body,
        {
        headers: new HttpHeaders({ 'Content-Type': 'application/x-www-form-urlencoded' }),
        withCredentials: true
        }).pipe(
        catchError(err => throwError(err))
    );
    }

getFormConfig(type: string, requestId: string): Observable<any> {
    return this.http.get<any>(this.formConfigUrl + type + '?request=' + requestId, {
        headers: new HttpHeaders({ 'Content-Type': 'application/x-www-form-urlencoded' }),
        withCredentials: true
        }).pipe(
        catchError(err => throwError(err))
    );
    }

You’re not supposed to send the POST request using AJAX / XHR but instead really do a regular POST request in the browser. I think that’s your problem!

To clarify: I mean when the user enters his/her password/username and then clicks on “Login” or “Sign Up”. This request should be a regular form request, so don’t hook into it with AJAX!

This was it. We changed the form to send natively rather than routing everything through JavaScript and it’s now working. Thank you for your help!