From 4b5d3e0e721a952f4ac9d17bee331e6dfe543dcd Mon Sep 17 00:00:00 2001
From: Lysandros Nikolaou <lisandrosnik@gmail.com>
Date: Wed, 12 Jun 2024 20:52:55 +0200
Subject: [PATCH] gh-120343: Fix column offsets of multiline tokens in tokenize
 (#120391)

---
 Lib/test/test_tokenize.py | 14 ++++++++++++++
 Python/Python-tokenize.c  | 14 ++++++++++----
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
index 36dba71766c..51aeb35f010 100644
--- a/Lib/test/test_tokenize.py
+++ b/Lib/test/test_tokenize.py
@@ -1210,6 +1210,20 @@ a = f'''
     FSTRING_END "\'\'\'"         (2, 68) (2, 71)
     """)
 
+    def test_multiline_non_ascii_fstring_with_expr(self):
+        self.check_tokenize("""\
+f'''
+    🔗 This is a test {test_arg1}🔗
+🔗'''""", """\
+    FSTRING_START "f\'\'\'"        (1, 0) (1, 4)
+    FSTRING_MIDDLE '\\n    🔗 This is a test ' (1, 4) (2, 21)
+    OP         '{'           (2, 21) (2, 22)
+    NAME       'test_arg1'   (2, 22) (2, 31)
+    OP         '}'           (2, 31) (2, 32)
+    FSTRING_MIDDLE '🔗\\n🔗'        (2, 32) (3, 1)
+    FSTRING_END "\'\'\'"         (3, 1) (3, 4)
+    """)
+
 class GenerateTokensTest(TokenizeTest):
     def check_tokenize(self, s, expected):
         # Format the tokens in s in a table format.
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
index 2591dae3573..55c821754c2 100644
--- a/Python/Python-tokenize.c
+++ b/Python/Python-tokenize.c
@@ -215,6 +215,7 @@ tokenizeriter_next(tokenizeriterobject *it)
 
     const char *line_start = ISSTRINGLIT(type) ? it->tok->multi_line_start : it->tok->line_start;
     PyObject* line = NULL;
+    int line_changed = 1;
     if (it->tok->tok_extra_tokens && is_trailing_token) {
         line = PyUnicode_FromString("");
     } else {
@@ -229,12 +230,11 @@ tokenizeriter_next(tokenizeriterobject *it)
             Py_XDECREF(it->last_line);
             line = PyUnicode_DecodeUTF8(line_start, size, "replace");
             it->last_line = line;
-            if (it->tok->lineno != it->last_end_lineno) {
-                it->byte_col_offset_diff = 0;
-            }
+            it->byte_col_offset_diff = 0;
         } else {
             // Line hasn't changed so we reuse the cached one.
             line = it->last_line;
+            line_changed = 0;
         }
     }
     if (line == NULL) {
@@ -252,7 +252,13 @@ tokenizeriter_next(tokenizeriterobject *it)
     Py_ssize_t byte_offset = -1;
     if (token.start != NULL && token.start >= line_start) {
         byte_offset = token.start - line_start;
-        col_offset = byte_offset - it->byte_col_offset_diff;
+        if (line_changed) {
+            col_offset = _PyPegen_byte_offset_to_character_offset_line(line, 0, byte_offset);
+            it->byte_col_offset_diff = byte_offset - col_offset;
+        }
+        else {
+            col_offset = byte_offset - it->byte_col_offset_diff;
+        }
     }
     if (token.end != NULL && token.end >= it->tok->line_start) {
         Py_ssize_t end_byte_offset = token.end - it->tok->line_start;