Add PLF postprocessor

configs/postprocessors/plf.yml

@@ -0,0 +1,24 @@
+postprocessor:
+  name: lpf
+  APS_mode: True
+  postprocessor_args:
+    y_start: -2.0
+    y_end: 5.0
+    delta_y: 1.0
+    q1: 0.40
+    u: 0.10
+    m1: 0.1
+    m2: -2.0
+
+  # PLF is trained with BO.
+  # For the BO setup, please refer to our codebase:
+  # https://github.com/maryhadalittlecode/PLF-OpenOOD/tree/master
+  # The grid search here is provided just as an example.
+  postprocessor_sweep:
+    y_start: [1]
+    y_end: [-1]
+    delta_y: [2]
+    q1: [0.62]
+    u: [0.84]
+    m1: [2]
+    m2: [0]

openood/evaluation_api/evaluator.py

@@ -13,6 +13,7 @@
 from openood.networks.ash_net import ASHNet
 from openood.networks.react_net import ReactNet
 from openood.networks.scale_net import ScaleNet
+from openood.networks.plf_net import PLFNet
 from openood.networks.adascale_net import AdaScaleANet, AdaScaleLNet
 
 from .datasets import DATA_INFO, data_setup, get_id_ood_dataloader
@@ -90,7 +91,8 @@ def __init__(
         # set up config root
         if config_root is None:
             filepath = os.path.dirname(os.path.abspath(__file__))
-            config_root = os.path.join('/', *filepath.split('/')[:-2], 'configs')
+            config_root = os.path.join('/',
+                                       *filepath.split('/')[:-2], 'configs')
 
         # get postprocessor
         if postprocessor is None:
@@ -121,6 +123,8 @@ def __init__(
             net = AdaScaleANet(net)
         elif postprocessor_name == 'adascale_l':
             net = AdaScaleLNet(net)
+        elif postprocessor_name == 'plf':
+            net = PLFNet(net)
 
         # postprocessor setup
         postprocessor.setup(net, dataloader_dict['id'], dataloader_dict['ood'])

openood/evaluation_api/postprocessor.py

@@ -16,8 +16,8 @@
     RMDSPostprocessor, SHEPostprocessor, CIDERPostprocessor, NPOSPostprocessor,
     GENPostprocessor, NNGuidePostprocessor, RelationPostprocessor,
     T2FNormPostprocessor, ReweightOODPostprocessor, fDBDPostprocessor,
-    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,CFOODPostprocessor,
-    VRAPostprocessor, GrOODPostprocessor)
+    AdaScalePostprocessor, IODINPostprocessor, NCIPostprocessor,
+    CFOODPostprocessor, VRAPostprocessor, GrOODPostprocessor, PLFPostprocessor)
 from openood.utils.config import Config, merge_configs
 
 postprocessors = {
@@ -73,9 +73,11 @@
     'grood': GrOODPostprocessor,
     'vra': VRAPostprocessor,
     'cfood': CFOODPostprocessor,
+    'plf': PLFPostprocessor
 }
 
-link_prefix = 'https://raw.githubusercontent.com/Jingkang50/OpenOOD/main/configs/postprocessors/'
+link_prefix = ('https://raw.githubusercontent.com/Jingkang50/'
+               'OpenOOD/main/configs/postprocessors/')
 
 
 def get_postprocessor(config_root: str, postprocessor_name: str,

openood/networks/plf_net.py

@@ -0,0 +1,44 @@
+import torch
+
+
+class PLFNet(torch.nn.Module):
+    def __init__(self, backbone):
+        super(PLFNet, self).__init__()
+        self.backbone = backbone
+
+    def forward(self, x, return_feature=False, return_feature_list=False):
+        try:
+            return self.backbone(x, return_feature, return_feature_list)
+        except TypeError:
+            return self.backbone(x, return_feature)
+
+    def forward_threshold(self, x, y_start, y_end, x1, y1, m1, x2, m2):
+        y_start = torch.tensor(y_start, device=x.device)
+        y_end = torch.tensor(y_end, device=x.device)
+        x1 = torch.tensor(x1, device=x.device)
+        y1 = torch.tensor(y1, device=x.device)
+        m1 = torch.tensor(m1, device=x.device)
+        x2 = torch.tensor(x2, device=x.device)
+        m2 = torch.tensor(m2, device=x.device)
+
+        _, features = self.backbone(x, return_feature=True)
+        x_abs = torch.abs(features)
+        m0 = (y_end - y_start) / x1
+
+        seg1 = y_start + m0 * x_abs
+        seg_jump = torch.full_like(x_abs, y1)
+        seg2 = y1 + m1 * (x_abs - x1)
+        seg3 = (y1 + m1 * (x2 - x1)) + m2 * (x_abs - x2)
+
+        y_pos = torch.where(
+            x_abs < x1, seg1,
+            torch.where(x_abs == x1, seg_jump,
+                        torch.where(x_abs <= x2, seg2, seg3)))
+        features = torch.where(features < 0, -y_pos, y_pos)
+        features = features.view(features.size(0), -1)
+        logits_cls = self.backbone.get_fc_layer()(features)
+        return logits_cls
+
+    def get_fc(self):
+        fc = self.backbone.fc
+        return fc.weight.cpu().detach().numpy(), fc.bias.cpu().detach().numpy()

openood/postprocessors/__init__.py

@@ -1,3 +1,4 @@
+# flake8: noqa
 from .nci_postprocessor import NCIPostprocessor
 from .fdbd_postprocessor import fDBDPostprocessor
 from .ash_postprocessor import ASHPostprocessor
@@ -50,4 +51,4 @@
 from .grood import GrOODPostprocessor
 from .vra_postprocessor import VRAPostprocessor
 from .cfood_postprocessor import CFOODPostprocessor
-
+from .plf_postprocessor import PLFPostprocessor

openood/postprocessors/plf_postprocessor.py

@@ -0,0 +1,103 @@
+from typing import Any
+
+import numpy as np
+import torch
+import torch.nn as nn
+from tqdm import tqdm
+
+from .base_postprocessor import BasePostprocessor
+
+
+class PLFPostprocessor(BasePostprocessor):
+    def __init__(self, config):
+        super(PLFPostprocessor, self).__init__(config)
+        self.args = self.config.postprocessor.postprocessor_args
+
+        # 7 hyperparameters for PLF postprocessor
+        self.y_start = self.args.y_start  # params 1
+        self.y_end = self.args.y_end  # params 2
+        self.q1 = self.args.q1
+        self.x1 = None  # calculated in setup based on q1 - params 3
+        self.y1 = None
+        # calculated in setup based on y_end + delta_y - params 4
+        self.m1 = self.args.m1  # params 5
+        self.q2 = None  # calculated in setup based on q1 and u (delta)
+        self.x2 = None  # calculated in setup based on q2 - params 6
+        self.m2 = self.args.m2  # params 7
+        self.delta_y = self.args.delta_y
+        self.u = self.args.u
+
+        self.args_dict = self.config.postprocessor.postprocessor_sweep
+        self.setup_flag = False
+
+    def setup(self, net: nn.Module, id_loader_dict, ood_loader_dict):
+        if not self.setup_flag:
+            activation_log = []
+            net.eval()
+            with torch.no_grad():
+                for batch in tqdm(id_loader_dict['val'],
+                                  desc='Setup: ',
+                                  position=0,
+                                  leave=True):
+                    data = batch['data'].cuda()
+                    data = data.float()
+
+                    _, feature = net(data, return_feature=True)
+                    activation_log.append(feature.data.cpu().numpy())
+
+            self.activation_log = np.concatenate(activation_log, axis=0)
+            self.setup_flag = True
+
+        else:
+            pass
+
+        self.y1 = self.y_end + self.delta_y
+        delta_min = 0.10
+        delta_max = 0.99 - self.q1
+        delta = delta_min + self.u * (delta_max - delta_min)
+        self.q2 = self.q1 + delta
+
+        sorted_id_feats = np.sort(np.abs(self.activation_log), axis=0)
+        n_id = sorted_id_feats.shape[0]
+
+        i1 = int(self.q1 * (n_id - 1))
+        i2 = int(self.q2 * (n_id - 1))
+        if i2 <= i1:
+            i2 = min(i1 + 1, n_id - 1)
+
+        self.x1 = sorted_id_feats[i1]
+        self.x2 = sorted_id_feats[i2]
+
+    @torch.no_grad()
+    def postprocess(self, net: nn.Module, data: Any):
+        output = net.forward_threshold(data, self.y_start, self.y_end, self.x1,
+                                       self.y1, self.m1, self.x2, self.m2)
+        score = torch.softmax(output, dim=1)
+        _, pred = torch.max(score, dim=1)
+        energyconf = torch.logsumexp(output.data.cpu(), dim=1)
+        return pred, energyconf
+
+    def set_hyperparam(self, hyperparam: list):
+        self.y_start, self.y_end, self.delta_y, \
+            self.q1, self.u, self.m1, self.m2 = hyperparam
+
+        self.y1 = self.y_end + self.delta_y
+        delta_min = 0.10
+        delta_max = 0.99 - self.q1
+        delta = delta_min + self.u * (delta_max - delta_min)
+        self.q2 = self.q1 + delta
+
+        sorted_id_feats = np.sort(np.abs(self.activation_log), axis=0)
+        n_id = sorted_id_feats.shape[0]
+
+        i1 = int(self.q1 * (n_id - 1))
+        i2 = int(self.q2 * (n_id - 1))
+        if i2 <= i1:
+            i2 = min(i1 + 1, n_id - 1)
+
+        self.x1 = sorted_id_feats[i1]
+        self.x2 = sorted_id_feats[i2]
+
+    def get_hyperparam(self):
+        return self.y_start, self.y_end, self.delta_y, \
+            self.q1, self.u, self.m1, self.m2

openood/postprocessors/utils.py

@@ -45,6 +45,7 @@
 from .relation_postprocessor import RelationPostprocessor
 from .grood import GrOODPostprocessor
 from .vra_postprocessor import VRAPostprocessor
+from .plf_postprocessor import PLFPostprocessor
 
 
 def get_postprocessor(config: Config):
@@ -94,6 +95,7 @@ def get_postprocessor(config: Config):
         't2fnorm': T2FNormPostprocessor,
         'grood': GrOODPostprocessor,
         'vra': VRAPostprocessor,
+        'plf': PLFPostprocessor
     }
 
     return postprocessors[config.postprocessor.name](config)

scripts/ood/plf/cifar100_test_ood_plf.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+# sh scripts/ood/ash/cifar100_test_ood_ash.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+# python main.py \
+#     --config configs/datasets/cifar100/cifar100.yml \
+#     configs/datasets/cifar100/cifar100_ood.yml \
+#     configs/networks/resnet18_32x32.yml \
+#     configs/pipelines/test/test_ood.yml \
+#     configs/preprocessors/base_preprocessor.yml \
+#     configs/postprocessors/ash.yml \
+#     --network.checkpoint 'results/cifar100_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt'
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar100 \
+    --root ./results/cifar100_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor plf \
+    --save-score --save-csv

scripts/ood/plf/cifar10_test_ood_plf.sh

@@ -0,0 +1,35 @@
+#!/bin/bash
+# sh scripts/ood/ash/cifar10_test_ood_ash.sh
+
+# GPU=1
+# CPU=1
+# node=73
+# jobname=openood
+
+PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+
+python main.py \
+    --config configs/datasets/cifar10/cifar10.yml \
+    configs/datasets/cifar10/cifar10_ood.yml \
+    configs/networks/resnet18_32x32.yml \
+    configs/pipelines/test/test_ood.yml \
+    configs/preprocessors/base_preprocessor.yml \
+    configs/postprocessors/ash.yml \
+    --num_workers 8 \
+    --network.checkpoint 'results/cifar10_resnet18_32x32_base_e100_lr0.1_default/s0/best.ckpt' \
+    --mark 1
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+python scripts/eval_ood.py \
+    --id-data cifar10 \
+    --root ./results/cifar10_resnet18_32x32_base_e100_lr0.1_default \
+    --postprocessor plf \
+    --save-score --save-csv

scripts/ood/plf/imagenet200_test_ood_plf.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+# sh scripts/ood/ash/imagenet200_test_ood_ash.sh
+
+############################################
+# alternatively, we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood.py
+# especially if you want to get results from
+# multiple runs
+
+# ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor plf \
+    --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood.py \
+    --id-data imagenet200 \
+    --root ./results/imagenet200_resnet18_224x224_base_e90_lr0.1_default \
+    --postprocessor plf \
+    --save-score --save-csv --fsood

scripts/ood/plf/imagenet_test_ood_plf.sh

@@ -0,0 +1,47 @@
+#!/bin/bash
+# sh scripts/ood/ash/imagenet_test_ood_ash.sh
+
+GPU=1
+CPU=1
+node=63
+jobname=openood
+
+# PYTHONPATH='.':$PYTHONPATH \
+# srun -p dsta --mpi=pmi2 --gres=gpu:${GPU} -n1 \
+# --cpus-per-task=${CPU} --ntasks-per-node=${GPU} \
+# --kill-on-bad-exit=1 --job-name=${jobname} -w SG-IDC1-10-51-2-${node} \
+# python main.py \
+#     --config configs/datasets/imagenet/imagenet.yml \
+#     configs/datasets/imagenet/imagenet_ood.yml \
+#     configs/networks/resnet50.yml \
+#     configs/pipelines/test/test_ood.yml \
+#     configs/preprocessors/base_preprocessor.yml \
+#     configs/postprocessors/ash.yml \
+#     --num_workers 4 \
+#     --ood_dataset.image_size 256 \
+#     --dataset.test.batch_size 256 \
+#     --dataset.val.batch_size 256 \
+#     --network.pretrained True \
+#     --network.checkpoint 'results/pretrained_weights/resnet50_imagenet1k_v1.pth' \
+#     --merge_option merge
+
+############################################
+# we recommend using the
+# new unified, easy-to-use evaluator with
+# the example script scripts/eval_ood_imagenet.py
+
+# available architectures:
+# resnet50, swin-t, vit-b-16
+# ood
+# python scripts/eval_ood_imagenet.py \
+#     --tvs-pretrained \
+#     --arch resnet50 \
+#     --postprocessor plf \
+#     --save-score --save-csv #--fsood
+
+# full-spectrum ood
+python scripts/eval_ood_imagenet.py \
+    --tvs-pretrained \
+    --arch resnet50 \
+    --postprocessor plf \
+    --save-score --save-csv --fsood