fvn-virt: ajoute support for synchro des volumes lvm, port de bscp vers python3

fcn-virt

@@ -2,28 +2,116 @@
 # Style Guide: https://www.python.org/dev/peps/pep-0008/
 
 import argparse
+import configparser
 import libvirt
+from pathlib import Path
+import subprocess
 import sys
+import time
+
+import fcntoolbox.bscp as bscp
+
+log_function = lambda x : print(x)
+log_file = sys.stdout
 
 parser = argparse.ArgumentParser()
-parser.add_argument("selection", type=str, choices=["is-active"])
-parser.add_argument("domain", type=str)
+parser.add_argument("selection", type=str, choices=["is-active", "is-volume-open", "vol-backup", "vol-serve"])
+parser.add_argument("--dom", "-d", type=str)
+parser.add_argument("--vol", type=str)
 
 args = parser.parse_args()
 
-def is_active(dom):
+config = configparser.RawConfigParser()
+config.read("/etc/fcntoolbox/config.ini")
+confvirt = config['virt']
+
+def is_active(dom_name):
   conn = libvirt.openReadOnly(None)
   if conn == None:
     print('Failed to open connection to the hypervisor')
     sys.exit(1)
 
-  dom = conn.lookupByName(args.domain)
+  dom = conn.lookupByName(dom_name)
   return dom.isActive()
 
+def is_volume_open(volpath):
+  if not Path(volpath).exists():
+    raise FileNotFoundError("Cannot find volume '{}'".format(volpath))
+  lvd_out = subprocess.check_output(['lvdisplay', volpath]).decode()
+  #   split lines v           v remove header
+  lvinfo = lvd_out.split("\n")[1:]
+  lvinfo = [line.strip() for line in lvinfo]
+  # Find line starting with "# open"
+  lvopen = list(filter(lambda line : line.find("# open") == 0, lvinfo))
+  # Read after "# open"
+  lvopen = lvopen[0][len("# open"):].strip()
+  return lvopen != "0"
+
+def create_backup_snapshot(vol_str):
+  snap_path = Path("{}_snap_{}".format(vol_str, time.strftime("%Y-%m-%d")))
+  snapshot_size = '10G'
+  subprocess.call(['lvcreate', '--snapshot', '--size', snapshot_size, '--name', snap_path.name, vol_str], stdout = log_file, stderr = log_file)
+  return snap_path
+
+def backup(dom_name, vol_str):
+  vol_path = Path(vol_str)
+  if not vol_path.exists():
+    raise FileNotFoundError("Cannot find volume '{}'".format(vol_str))
+  
+  conn = libvirt.open(None)
+  if conn == None:
+    print('Failed to open connection to the hypervisor')
+    sys.exit(1)
+  
+  dom = conn.lookupByName(dom_name)
+  is_active = dom.isActive()
+  if is_active:
+    #try:
+    #  dom.fSTrim(None, 4096)
+    #except libvirt.libvirtError as e:
+    #  print(e)
+    dom.suspend()
+  
+  snap_path = create_backup_snapshot(vol_str)
+  
+  if is_active:
+    dom.resume()
+  
+  remote_vol_str = '/dev/{}/{}'.format(confvirt['remote_vg'], vol_path.name)
+  bscp.remote_command = confvirt['remote_command']
+  bscp.bscp(str(snap_path), confvirt['remote_host'], remote_vol_str, 64 * 1024, 'sha1')
+
+def bscp_device_function(filename):
+  allowed_volumes = set(confvirt['allowed_volumes'].split(','))
+  if not filename in allowed_volumes:
+    return None
+  filename = str(create_backup_snapshot(filename))
+  if is_volume_open(filename):
+    return None
+  else:
+    return filename
+
 if args.selection == 'is-active':
-  if is_active(args.domain):
+  if args.dom is None:
+    parser.error("is-active requires --domain.")
+  if is_active(args.dom):
     sys.exit(0)
   else:
     sys.exit(1)
-
+elif args.selection == 'is-volume-open':
+  if args.vol is None:
+    parser.error("is-volume-open requires --vol.")
+  if is_volume_open(args.vol):
+    sys.exit(0)
+  else:
+    sys.exit(1)
+elif args.selection == 'vol-backup':
+  if args.dom is None or args.vol is None:
+    parser.error("vol-backup requires --domain and --vol.")
+  backup(args.dom, args.vol)
+elif args.selection == 'vol-serve':
+  log_function = lambda x : None
+  log_file = subprocess.DEVNULL
+  bscp.device_function =  bscp_device_function
+  bscp.serve()
 

fcntoolbox/bscp.py

@@ -0,0 +1,177 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2016
+#
+# * Volker Diels-Grabsch <v@njh.eu>
+# * art0int <zvn_mail@mail.ru>
+# * guillaume <guillaume@atto.be>
+#
+# Permission to use, copy, modify, and/or distribute this software for any
+# purpose with or without fee is hereby granted, provided that the above
+# copyright notice and this permission notice appear in all copies.
+#
+# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+# ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+# ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+# OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+import hashlib
+import os
+import os.path
+import struct
+import subprocess
+import sys
+
+# device_function: filters or transforms device names
+device_function = lambda dev : dev
+
+# remote_command: command to run bscp on the remote host
+remote_command = __file__
+
+def serve():
+    global allowed_devices
+    (size, blocksize, filename_len, hashname_len) = struct.unpack('<QQQQ', sys.stdin.buffer.read(8+8+8+8))
+    filename = sys.stdin.buffer.read(filename_len).decode()
+    hashname = sys.stdin.buffer.read(hashname_len).decode()
+    
+    filename = device_function(filename)
+
+    if filename == None or len(filename) == 0:
+        return
+
+    if not os.path.exists(filename):
+        # Create sparse file
+        with open(filename, 'wb') as f:
+            f.truncate(size)
+        os.chmod(filename, 0o600)
+
+    with open(filename, 'rb+') as f:
+        f.seek(0, 2)
+        sys.stdout.buffer.write(struct.pack('<Q', f.tell()))
+        readremain = size
+        rblocksize = blocksize
+        f.seek(0)
+        while True:
+            if readremain <= blocksize:
+                rblocksize = readremain
+            block = f.read(rblocksize)
+            if len(block) == 0:
+                break
+            digest = hashlib.new(hashname, block).digest()
+            sys.stdout.buffer.write(digest)
+            readremain -= rblocksize
+            if readremain == 0:
+                break
+        sys.stdout.flush()
+        while True:
+            position_s = sys.stdin.buffer.read(8)
+            if len(position_s) == 0:
+                break
+            (position,) = struct.unpack('<Q', position_s)
+            block = sys.stdin.buffer.read(blocksize)
+            f.seek(position)
+            f.write(block)
+        readremain = size
+        rblocksize = blocksize
+        hash_total = hashlib.new(hashname)
+        f.seek(0)
+        while True:
+            if readremain <= blocksize:
+                rblocksize = readremain
+            block = f.read(rblocksize)
+            if len(block) == 0:
+                break
+            hash_total.update(block)
+            readremain -= rblocksize
+            if readremain == 0:
+                break
+    sys.stdout.buffer.write(hash_total.digest())
+
+class IOCounter:
+    def __init__(self, in_stream, out_stream):
+        self.in_stream = in_stream
+        self.out_stream = out_stream
+        self.in_total = 0
+        self.out_total = 0
+    def read(self, size=None):
+        if size is None:
+            s = self.in_stream.read()
+        else:
+            s = self.in_stream.read(size)
+        self.in_total += len(s)
+        return s
+    def write(self, s):
+        self.out_stream.write(s)
+        self.out_total += len(s)
+    def flush(self):
+        self.out_stream.flush()
+
+def bscp(local_filename, remote_host, remote_filename, blocksize, hashname):
+    hash_total = hashlib.new(hashname)
+    with open(local_filename, 'rb') as f:
+        f.seek(0, 2)
+        size = f.tell()
+        f.seek(0)
+
+        # Calculate number of blocks, including the last block which may be smaller
+        blockcount = int((size + blocksize - 1) / blocksize)
+
+        command = ('ssh', remote_host, '--', remote_command)
+        p = subprocess.Popen(command, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+        io = IOCounter(p.stdout, p.stdin)
+
+        remote_filename_bytes = remote_filename.encode()
+        hashname_bytes = hashname.encode()
+        io.write(struct.pack('<QQQQ', size, blocksize, len(remote_filename_bytes), len(hashname_bytes)))
+        io.write(remote_filename_bytes)
+        io.write(hashname_bytes)
+        io.flush()
+
+        (remote_size,) = struct.unpack('<Q', io.read(8))
+        if remote_size < size:
+            raise RuntimeError('Remote size less than local (local: %i, remote: %i)' % (size, remote_size))
+        remote_digest_list = [io.read(hash_total.digest_size) for i in range(blockcount)]
+
+        for remote_digest in remote_digest_list:
+            position = f.tell()
+            block = f.read(blocksize)
+            hash_total.update(block)
+            digest = hashlib.new(hashname, block).digest()
+            if digest != remote_digest:
+                try:
+                    io.write(struct.pack('<Q', position))
+                    io.write(block)
+                except IOError:
+                    break
+        io.flush()
+        p.stdin.close()
+
+        remote_digest_total = io.read()
+        p.wait()
+        if remote_digest_total != hash_total.digest():
+            raise RuntimeError('Checksum mismatch after transfer')
+    return (io.in_total, io.out_total, size)
+
+if __name__ == '__main__':
+    try:
+        local_filename = sys.argv[1]
+        (remote_host, remote_filename) = sys.argv[2].split(':')
+        if len(sys.argv) >= 4:
+            blocksize = int(sys.argv[3])
+        else:
+            blocksize = 64 * 1024
+        if len(sys.argv) >= 5:
+            hashname = sys.argv[4]
+        else:
+            hashname = 'sha1'
+        assert len(sys.argv) <= 5
+    except:
+        usage = 'bscp SRC HOST:DEST [BLOCKSIZE] [HASH]'
+        sys.stderr.write('Usage:\n\n    %s\n\n' % (usage,))
+        sys.exit(1)
+    (in_total, out_total, size) = bscp(local_filename, remote_host, remote_filename, blocksize, hashname)
+    speedup = size * 1.0 / (in_total + out_total)
+    sys.stderr.write('in=%i out=%i size=%i speedup=%.2f\n' % (in_total, out_total, size, speedup))