This code can just change the pattern to (cat.{0,10}(red|blue))|(white|black)|cats.{0,10}dogs, which is a little bit different from what you want. Would this be helpful?

import re


def find_ending(text='', content='', begin=None):
    if not text or not content or content.find(text) == -1:
        return -1

    if text not in symbols or symbols[text] not in content:
        return -1

    end_text = symbols[text]
    begin = (content.find(text) if begin is None else begin) + 1

    count = 0
    content = content[begin:]

    reg = re.compile(r'[%s]' % re.escape(text + end_text))

    for r in reg.finditer(content):
        if r[0] == text:
            count += 1
        else:
            count -= 1

        if count == -1:
            return r.start() + begin + len(r.group(0))
    else:
        return -1


pattern = r'dogs.{0,10}cats|(white|black)|(cat.{0,10}(red|blue))'

reg = re.compile(r'\w+|(\.\{\d+,\d+\})|(\([^\)]+\))|\|')
symbols = {
    '{': '}',
    '(': ')'
}
results = []
match = reg.search(pattern)
end = 0
while match:
    if len(match[0]) != 1 and match[0][0] in symbols:
        end = find_ending(match[0][0], pattern)
        assert end != -1  # raise AssertError if can not find an ending position of current symbol
        results.append(pattern[match.start():end])
    else:
        results.append(match[0])
        end = len(match[0])
    pattern = pattern[end:]
    match = reg.search(pattern)

print(''.join(results[::-1]))  # (cat.{0,10}(red|blue))|(white|black)|cats.{0,10}dogs