That should do the job (untested):

jobs:
  respond-to-label:
    if: |
      (github.event.action == 'labeled' && contains(github.event.label.name, 'run-special-ci')) ||
      (github.event.action == 'unlabeled' && contains(github.event.label.name, 'not-run-ci'))
    runs-on: ubuntu-latest