INFN wiki

User Tools

Site Tools

Trace:
progetti:cloud-areapd:operations:manual_delete_cinder_volumes

Differences

This shows you the differences between two versions of the page.

Link to this comparison view

Both sides previous revisionPrevious revision
Next revision		Previous revision
progetti:cloud-areapd:operations:manual_delete_cinder_volumes [2015/06/18 07:07] – dorigoa@infn.itprogetti:cloud-areapd:operations:manual_delete_cinder_volumes [2017/05/22 15:09] (current) – fanzago@infn.it
Line 1: Line 1:
 +====== Manual delete Cinder Volumes ======
  
 +Author:
 +   * Alvise Dorigo (INFN Padova)
 +
 +
 +There are cases, not easily reproducible, in which ''cinder delete'' or even ''cinder force-delete'' do not work, and the volume stays in its current status (which is, in our experience, "deleting" or "detaching"...) forever (digging into the cinder's log files is almost useless).
 +
 +The following procedure (a perl script) erases any information about a problematic volume by "brute force", i.e. by modifying the cinder's database table. Then, it must be used as **very last option**.
 +
 +**Note1**: the procedure isn't flawless; it only touches the cinder's table (''cinder_prod'' database's tables in our case) so in theory it should be harmful for the correct OpenStack working. But I've seen that if, for example, the ''quota_usages'' table is not correctly updated (e.g. by putting negative values for a certain column) the login in the dashboard stops working! presumably because the Overview window (which is the first invoked just after the login) gets an un-handled error with negative values.
 +Of course, the script checks that non-negative values are put in the ''quota_usages'' table. But I don't know yet if there other scenarios that could produce errors like this.
 +So the suggestion is to do a precautionary backup of the ''cinder_prod'' database before running this script.
 +
 +**Note2**: the script doesn't remove the physical file related to the cinder volume. It must be erased by hand (and the related snapshots and .info files too)
 +
 +**Note3**: in the script pasted below, the DB_HOSTNAME, DB_CINDERUSERNAME, DB_CINDERPWD, DB_CINDER labels must be replaced by the proper values.
 +
 +**Note4**: to run the script you need to install ''perl-DBI'' and ''perl-DBD-MySQL'' and, of course, the perl interpreter.
 +
 +<code perl>
 +#/usr/bin/perl
 +
 +use DBI;
 +
 +$VOLUME_UUID = $ARGV[0];
 +if($VOLUME_UUID eq "")
 +{ 
 +  print "usage: $0 <VOLUME_ID>\n";
 +  exit 1;
 +};
 +
 +$dsn = "DBI:mysql:database=DB_CINDER;host=DB_HOSTNAME;port=3306";
 +
 +$dbh = DBI->connect($dsn, "DB_CINDERUSERNAME", "DB_CINDERPWD") or die "Couldn't connect to database: " . DBI->errstr;
 +
 +#
 +# Get the project ID and size of the volume to delete
 +#
 +$query_get_project_id_from_volume = "SELECT project_id,size,display_name,status,deleted FROM volumes WHERE id='$VOLUME_UUID';";
 +$sth = $dbh->prepare( $query_get_project_id_from_volume );
 +$sth->execute( );
 +$ref = $sth->fetchrow_hashref( );
 +$size = $ref->{'size'};
 +$tenant = $ref->{'project_id'};
 +$volname = $ref->{'display_name'};
 +$status = $ref->{'status'};
 +$deleted = $ref->{'deleted'};
 +if($status eq "deleted" && $deleted eq '1') {
 +  print "The volume [$VOLUME_UUID] seems already deleted. Stop.\n";
 +  exit 0;
 +}
 +
 +print "Volume [$VOLUME_UUID] name [$volname] size [$size] GB belongs to project ID [$tenant]\n";
 +$sth->finish( );
 +
 +#
 +# Get the number of snapshots related to the current volume to delete (they must be deleted also)
 +#
 +$count_snap_related_to_volume_query = "SELECT count(id) FROM snapshots WHERE volume_id='$VOLUME_UUID';";
 +$sth = $dbh->prepare( $count_snap_related_to_volume_query );
 +$sth->execute( );
 +@rows = $sth->fetchrow_array( );
 +$num_of_snapshots = $rows[0];
 +$sth->finish( );
 +
 +#
 +# Prepare delete and quota update queries
 +#
 +$mark_voldeleted_query       = "UPDATE volumes SET updated_at=now(),deleted_at=now(),terminated_at=now(),status='deleted',deleted=1,attach_status='detached' WHERE id='$VOLUME_UUID';";
 +$mark_snapdeleted_query      = "UPDATE snapshots SET updated_at=now(),deleted_at=now(),status='deleted',deleted=1 WHERE volume_id='$VOLUME_UUID';";
 +$get_quota_usage_gb_query    = "SELECT in_use FROM quota_usages WHERE resource='gigabytes' AND project_id='$tenant';";
 +$get_quota_usage_vol_query   = "SELECT in_use FROM quota_usages WHERE resource='volumes' AND project_id='$tenant';";
 +$get_quota_usage_snaps_query = "SELECT in_use FROM quota_usages WHERE resource='snapshots' AND project_id='$tenant';";
 +print "Going to modify cinder_prod.volumes and cinder_prod.snapshots tables. Are you sure ? Press <Enter> or <Return> to continue, CTRL-C to exit: ";
 +my $resp = <STDIN>;
 +#
 +# First delete snapshot
 +#
 +$sth = $dbh->prepare( $mark_snapdeleted_query );
 +$sth->execute( );
 +$sth->finish( );
 +
 +#
 +# Second delete volume
 +#
 +$sth = $dbh->prepare( $mark_voldeleted_query );
 +$sth->execute( );
 +$sth->finish( );
 +
 +#
 +# Get info about Volume GB quota usage (which includes the snapshots)
 +#
 +$sth = $dbh->prepare( $get_quota_usage_gb_query );
 +$sth->execute( );
 +@result = $sth->fetchrow_array( );
 +$in_use_gb = $result[0];
 +$sth->finish( );
 +
 +#
 +# Get the number of volumes
 +#
 +$sth = $dbh->prepare( $get_quota_usage_vol_query );
 +$sth->execute( );
 +@result = $sth->fetchrow_array( );
 +$in_use_vol = $result[0];
 +$sth->finish( );
 +
 +#
 +# Get the number of snapshots
 +#
 +$sth = $dbh->prepare( $get_quota_usage_snaps_query );
 +$sth->execute( );
 +@result = $sth->fetchrow_array( );
 +$in_use_snaps= $result[0];
 +$sth->finish( );
 +
 +#print "$in_use_gb GB - $in_use_vol volumes - $in_use_snaps snapshots\n";
 +
 +$in_use_vol--;
 +$in_use_gb -= $size + ($size*$num_of_snapshots);
 +$in_use_snaps -= $num_of_snapshots;
 +
 +if($in_use_vol<0) {
 +  $in_use_vol = 0;
 +}
 +if($in_use_gb<0) {
 +  $in_use_gb = 0;
 +}
 +if($in_use_snaps<0) {
 +  $in_use_snaps = 0;
 +}
 +
 +#print "$in_use_gb GB - $in_use_vol volumes - $in_use_snaps snapshots\n";
 +
 +$update_quota_vol   = "UPDATE quota_usages SET in_use='$in_use_vol' WHERE resource='volumes' AND project_id='$tenant';";
 +$update_quota_gb    = "UPDATE quota_usages SET in_use='$in_use_gb' WHERE resource='gigabytes' AND project_id='$tenant';";
 +$update_quota_snaps = "UPDATE quota_usages SET in_use='$in_use_snaps' WHERE resource='snapshots' AND project_id='$tenant';";
 +$sth = $dbh->prepare( $update_quota_vol );
 +$sth->execute( );
 +$sth->finish();
 +$sth = $dbh->prepare( $update_quota_gb );
 +$sth->execute( );
 +$sth->finish();
 +$sth = $dbh->prepare( $update_quota_snaps );
 +$sth->execute( );
 +$sth->finish();
 +
 +$dbh->disconnect( );
 +</code>
Donate Powered by PHP Valid HTML5 Valid CSS Driven by DokuWiki