Author:

• Alvise Dorigo (INFN Padova)

There are cases, not easily reproducible, in which cinder delete or even cinder force-delete do not work, and the volume stays in its current status (which is, in our experience, "deleting" or "detaching"…) forever (digging into the cinder's log files is almost useless).

The following procedure (a perl script) erases any information about a problematic volume by "brute force", i.e. by modifying the cinder's database table. Then, it must be used as very last option.

Note1: the procedure isn't flawless; it only touches the cinder's table (cinder_prod database's tables in our case) so in theory it should be harmful for the correct OpenStack working. But I've seen that if, for example, the quota_usages table is not correctly updated (e.g. by putting negative values for a certain column) the login in the dashboard stops working! presumably because the Overview window (which is the first invoked just after the login) gets an un-handled error with negative values. Of course, the script checks that non-negative values are put in the quota_usages table. But I don't know yet if there other scenarios that could produce errors like this. So the suggestion is to do a precautionary backup of the cinder_prod database before running this script.

Note2: the script doesn't remove the physical file related to the cinder volume. It must be erased by hand (and the related snapshots and .info files too)

Note3: in the script pasted below, the DB_HOSTNAME, DB_CINDERUSERNAME, DB_CINDERPWD, DB_CINDER labels must be replaced by the proper values.

Note4: to run the script you need to install perl-DBI and perl-DBD-MySQL and, of course, the perl interpreter.

#/usr/bin/perl
 
use DBI;
 
$VOLUME_UUID = $ARGV[0];
if($VOLUME_UUID eq "")
{ 
  print "usage: $0 <VOLUME_ID>\n";
  exit 1;
};
 
$dsn = "DBI:mysql:database=DB_CINDER;host=DB_HOSTNAME;port=3306";
 
$dbh = DBI->connect($dsn, "DB_CINDERUSERNAME", "DB_CINDERPWD") or die "Couldn't connect to database: " . DBI->errstr;
 
#
# Get the project ID and size of the volume to delete
#
$query_get_project_id_from_volume = "SELECT project_id,size,display_name,status,deleted FROM volumes WHERE id='$VOLUME_UUID';";
$sth = $dbh->prepare( $query_get_project_id_from_volume );
$sth->execute( );
$ref = $sth->fetchrow_hashref( );
$size = $ref->{'size'};
$tenant = $ref->{'project_id'};
$volname = $ref->{'display_name'};
$status = $ref->{'status'};
$deleted = $ref->{'deleted'};
if($status eq "deleted" && $deleted eq '1') {
  print "The volume [$VOLUME_UUID] seems already deleted. Stop.\n";
  exit 0;
}
 
print "Volume [$VOLUME_UUID] name [$volname] size [$size] GB belongs to project ID [$tenant]\n";
$sth->finish( );
 
#
# Get the number of snapshots related to the current volume to delete (they must be deleted also)
#
$count_snap_related_to_volume_query = "SELECT count(id) FROM snapshots WHERE volume_id='$VOLUME_UUID';";
$sth = $dbh->prepare( $count_snap_related_to_volume_query );
$sth->execute( );
@rows = $sth->fetchrow_array( );
$num_of_snapshots = $rows[0];
$sth->finish( );
 
#
# Prepare delete and quota update queries
#
$mark_voldeleted_query       = "UPDATE volumes SET updated_at=now(),deleted_at=now(),terminated_at=now(),status='deleted',deleted=1,attach_status='detached' WHERE id='$VOLUME_UUID';";
$mark_snapdeleted_query      = "UPDATE snapshots SET updated_at=now(),deleted_at=now(),status='deleted',deleted=1 WHERE volume_id='$VOLUME_UUID';";
$get_quota_usage_gb_query    = "SELECT in_use FROM quota_usages WHERE resource='gigabytes' AND project_id='$tenant';";
$get_quota_usage_vol_query   = "SELECT in_use FROM quota_usages WHERE resource='volumes' AND project_id='$tenant';";
$get_quota_usage_snaps_query = "SELECT in_use FROM quota_usages WHERE resource='snapshots' AND project_id='$tenant';";
print "Going to modify cinder_prod.volumes and cinder_prod.snapshots tables. Are you sure ? Press <Enter> or <Return> to continue, CTRL-C to exit: ";
my $resp = <STDIN>;
#
# First delete snapshot
#
$sth = $dbh->prepare( $mark_snapdeleted_query );
$sth->execute( );
$sth->finish( );
 
#
# Second delete volume
#
$sth = $dbh->prepare( $mark_voldeleted_query );
$sth->execute( );
$sth->finish( );
 
#
# Get info about Volume GB quota usage (which includes the snapshots)
#
$sth = $dbh->prepare( $get_quota_usage_gb_query );
$sth->execute( );
@result = $sth->fetchrow_array( );
$in_use_gb = $result[0];
$sth->finish( );
 
#
# Get the number of volumes
#
$sth = $dbh->prepare( $get_quota_usage_vol_query );
$sth->execute( );
@result = $sth->fetchrow_array( );
$in_use_vol = $result[0];
$sth->finish( );
 
#
# Get the number of snapshots
#
$sth = $dbh->prepare( $get_quota_usage_snaps_query );
$sth->execute( );
@result = $sth->fetchrow_array( );
$in_use_snaps= $result[0];
$sth->finish( );
 
#print "$in_use_gb GB - $in_use_vol volumes - $in_use_snaps snapshots\n";
 
$in_use_vol--;
$in_use_gb -= $size + ($size*$num_of_snapshots);
$in_use_snaps -= $num_of_snapshots;
 
if($in_use_vol<0) {
  $in_use_vol = 0;
}
if($in_use_gb<0) {
  $in_use_gb = 0;
}
if($in_use_snaps<0) {
  $in_use_snaps = 0;
}
 
#print "$in_use_gb GB - $in_use_vol volumes - $in_use_snaps snapshots\n";
 
$update_quota_vol   = "UPDATE quota_usages SET in_use='$in_use_vol' WHERE resource='volumes' AND project_id='$tenant';";
$update_quota_gb    = "UPDATE quota_usages SET in_use='$in_use_gb' WHERE resource='gigabytes' AND project_id='$tenant';";
$update_quota_snaps = "UPDATE quota_usages SET in_use='$in_use_snaps' WHERE resource='snapshots' AND project_id='$tenant';";
$sth = $dbh->prepare( $update_quota_vol );
$sth->execute( );
$sth->finish();
$sth = $dbh->prepare( $update_quota_gb );
$sth->execute( );
$sth->finish();
$sth = $dbh->prepare( $update_quota_snaps );
$sth->execute( );
$sth->finish();
 
$dbh->disconnect( );