added multi-node locked ip search

2019-07-08 13:01:38 -04:00 · 2019-07-08 13:01:38 -04:00 · ae0b85f235
parent a83d00456b
commit ae0b85f235
1 changed files with 3 additions and 1 deletions
--- a/pytorch_lightning/models/trainer.py
+++ b/pytorch_lightning/models/trainer.py
@ -307,7 +307,9 @@ class Trainer(TrainerIO):
            my_ip = subprocess.run(['hostname', '-I'], stdout=subprocess.PIPE).stdout.decode('utf-8')
            my_ip = my_ip.split(' ')[0]

-            test_name = f'{os.getpid()}_{my_ip}'
+            r = np.random.uniform(0, 1, 1)[0]
+
+            test_name = f'{os.getpid()}_{my_ip}_{r}'
            ip_dir = os.path.join(self.exp_save_path, '.ips', test_name)
            os.makedirs(ip_dir, exist_ok=True)